mirror of
https://github.com/outbackdingo/firezone.git
synced 2026-01-27 10:18:54 +00:00
6a538368cb
Network flow logs are a common feature of VPNs. Due to the nature of a shared exit node, it is of great interest to a network analyst, which TCP connections are getting routed through the tunnel, who is initiating them, for long do they last and how much traffic is sent across them. With this PR, the Firezone Gateway gains the ability of detecting the TCP and UDP flows that are being routed through it. The information we want to attach to these flows is spread out over several layers of the packet handling code. To simplify the implementation and not complicate the APIs unnecessarily, we chose to rely on TLS (thread-local storage) for gathering all the necessary data as a packet gets passed through the various layers. When using a const initializer, the overhead of a TLS variable over an actual local variable is basically zero. The entire routing state of the Gateway is also never sent across any threads, making TLS variables a particularly good choice for this problem. In its MVP form, the detected flows are only emitted on stdout and also that only if `flow_logs=trace` is set using `RUST_LOG`. Early adopters of this feature are encouraged to enable these logs as described and then ingest the Gateway's logs into the SIEM of their choice for further analysis. Related: #8353
521 lines
14 KiB
YAML
521 lines
14 KiB
YAML
# Run with DOCKER_BUILD_TARGET=dev to build Rust inside Docker
|
|
include:
|
|
- scripts/compose/resources.yml
|
|
- scripts/compose/edgeshark.yml
|
|
- scripts/compose/portal.yml
|
|
|
|
x-erlang-cluster: &erlang-cluster
|
|
ERLANG_CLUSTER_ADAPTER: "Elixir.Cluster.Strategy.Epmd"
|
|
ERLANG_CLUSTER_ADAPTER_CONFIG: '{"hosts":["api@api.cluster.local","web@web.cluster.local","domain@domain.cluster.local"]}'
|
|
|
|
x-portal-urls: &portal-urls
|
|
WEB_EXTERNAL_URL: http://localhost:8080/
|
|
API_EXTERNAL_URL: http://localhost:8081/
|
|
|
|
x-phoenix-config: &phoenix-config
|
|
PHOENIX_HTTP_WEB_PORT: "8080"
|
|
PHOENIX_HTTP_API_PORT: "8081"
|
|
PHOENIX_SECURE_COOKIES: "false"
|
|
|
|
x-health-check: &health-check
|
|
interval: 1s
|
|
retries: 15
|
|
timeout: 1s
|
|
|
|
services:
|
|
web:
|
|
extends:
|
|
file: scripts/compose/portal.yml
|
|
service: common
|
|
build:
|
|
context: elixir
|
|
args:
|
|
APPLICATION_NAME: web
|
|
image: ${WEB_IMAGE:-ghcr.io/firezone/web}:${WEB_TAG:-main}
|
|
hostname: web.cluster.local
|
|
ports:
|
|
- 8080:8080/tcp
|
|
environment:
|
|
<<: [*portal-urls, *erlang-cluster, *phoenix-config]
|
|
RELEASE_HOSTNAME: "web.cluster.local"
|
|
RELEASE_NAME: "web"
|
|
LOG_LEVEL: "debug"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f localhost:8080/healthz"]
|
|
<<: *health-check
|
|
depends_on:
|
|
vault:
|
|
condition: "service_healthy"
|
|
postgres:
|
|
condition: "service_healthy"
|
|
|
|
api:
|
|
extends:
|
|
file: scripts/compose/portal.yml
|
|
service: common
|
|
build:
|
|
context: elixir
|
|
args:
|
|
APPLICATION_NAME: api
|
|
image: ${API_IMAGE:-ghcr.io/firezone/api}:${API_TAG:-main}
|
|
hostname: api.cluster.local
|
|
ports:
|
|
- 8081:8081/tcp
|
|
environment:
|
|
<<: [*portal-urls, *erlang-cluster, *phoenix-config]
|
|
RELEASE_HOSTNAME: "api.cluster.local"
|
|
RELEASE_NAME: "api"
|
|
LOG_LEVEL: "debug"
|
|
user: root # Needed to run `ip route` commands
|
|
cap_add:
|
|
- NET_ADMIN # Needed to run `tc` commands to add simulated delay
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
|
|
# Add static route to internet subnet via router
|
|
ip -4 route add 203.0.113.0/24 via 172.28.0.254
|
|
ip -6 route add 203:0:113::/64 via 172:28:0::254
|
|
|
|
exec su default -c "bin/server"
|
|
depends_on:
|
|
vault:
|
|
condition: "service_healthy"
|
|
postgres:
|
|
condition: "service_healthy"
|
|
api-router:
|
|
condition: "service_healthy"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f localhost:8081/healthz"]
|
|
<<: *health-check
|
|
networks:
|
|
app-internal:
|
|
ipv4_address: 172.28.0.100
|
|
ipv6_address: 172:28:0::100
|
|
|
|
api-router:
|
|
extends:
|
|
file: scripts/compose/router.yml
|
|
service: router
|
|
environment:
|
|
PORT_FORWARDS: |
|
|
8081 172.28.0.100 tcp
|
|
8081 172:28:0::100 tcp
|
|
MASQUERADE_TYPE: ""
|
|
NETWORK_LATENCY_MS: 50
|
|
networks:
|
|
app-internal:
|
|
ipv4_address: 172.28.0.254
|
|
ipv6_address: 172:28:0::254
|
|
interface_name: internal
|
|
internet:
|
|
ipv4_address: 203.0.113.10
|
|
ipv6_address: 203:0:113::10
|
|
interface_name: internet
|
|
|
|
domain:
|
|
extends:
|
|
file: scripts/compose/portal.yml
|
|
service: common
|
|
build:
|
|
context: elixir
|
|
args:
|
|
APPLICATION_NAME: domain
|
|
image: ${DOMAIN_IMAGE:-ghcr.io/firezone/domain}:${DOMAIN_TAG:-main}
|
|
hostname: domain.cluster.local
|
|
environment:
|
|
<<: *erlang-cluster
|
|
RELEASE_HOSTNAME: "domain.cluster.local"
|
|
RELEASE_NAME: "domain"
|
|
LOG_LEVEL: "debug"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f localhost:4000/healthz"]
|
|
<<: *health-check
|
|
depends_on:
|
|
vault:
|
|
condition: "service_healthy"
|
|
postgres:
|
|
condition: "service_healthy"
|
|
|
|
# This is a service container which allows to run mix tasks for local development
|
|
# without having to install Elixir and Erlang on the host machine.
|
|
elixir:
|
|
extends:
|
|
file: scripts/compose/portal.yml
|
|
service: common
|
|
build:
|
|
context: elixir
|
|
target: compiler
|
|
args:
|
|
APPLICATION_NAME: api
|
|
image: ${ELIXIR_IMAGE:-ghcr.io/firezone/elixir}:${ELIXIR_TAG:-main}
|
|
hostname: elixir
|
|
environment:
|
|
<<: *portal-urls
|
|
RELEASE_HOSTNAME: "mix.cluster.local"
|
|
RELEASE_NAME: "mix"
|
|
# Higher log level not to make seeds output too verbose
|
|
LOG_LEVEL: "info"
|
|
# Mix env should be set to prod to use secrets declared above,
|
|
# otherwise seeds will generate invalid tokens
|
|
MIX_ENV: "prod"
|
|
depends_on:
|
|
postgres:
|
|
condition: "service_healthy"
|
|
|
|
# Run with DOCKER_BUILD_TARGET=dev to build Rust inside Docker
|
|
client:
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "ip link | grep tun-firezone"]
|
|
<<: *health-check
|
|
environment:
|
|
FIREZONE_DNS_CONTROL: "${FIREZONE_DNS_CONTROL:-etc-resolv-conf}"
|
|
FIREZONE_TOKEN: "n.SFMyNTY.g2gDaANtAAAAJGM4OWJjYzhjLTkzOTItNGRhZS1hNDBkLTg4OGFlZjZkMjhlMG0AAAAkN2RhN2QxY2QtMTExYy00NGE3LWI1YWMtNDAyN2I5ZDIzMGU1bQAAACtBaUl5XzZwQmstV0xlUkFQenprQ0ZYTnFJWktXQnMyRGR3XzJ2Z0lRdkZnbgYAR_ywiZQBYgABUYA.PLNlzyqMSgZlbQb1QX5EzZgYNuY9oeOddP0qDkTwtGg"
|
|
RUST_LOG: ${RUST_LOG:-wire=trace,debug}
|
|
FIREZONE_API_URL: ws://api:8081
|
|
FIREZONE_ID: EFC7A9E3-3576-4633-B633-7D47BA9E14AC
|
|
FZFF_ICMP_ERROR_UNREACHABLE_PROHIBITED_CREATE_NEW_FLOW: true
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
|
|
# Add static route to internet subnet via router
|
|
ip -4 route add 203.0.113.0/24 via 172.30.0.254
|
|
ip -6 route add 203:0:113::/64 via 172:30:0::254
|
|
|
|
# Disable checksum offloading so that checksums are correct when they reach the relay
|
|
apk add --no-cache ethtool
|
|
ethtool -K eth0 tx off
|
|
|
|
exec firezone-headless-client
|
|
init: true
|
|
build:
|
|
target: ${DOCKER_BUILD_TARGET:-debug}
|
|
context: rust
|
|
dockerfile: Dockerfile
|
|
args:
|
|
PACKAGE: firezone-headless-client
|
|
image: ${CLIENT_IMAGE:-ghcr.io/firezone/debug/client}:${CLIENT_TAG:-main}
|
|
privileged: true # Needed to tune `sysctl` inside container.
|
|
cap_add:
|
|
- NET_ADMIN
|
|
sysctls:
|
|
- net.ipv6.conf.all.disable_ipv6=0
|
|
- net.ipv6.conf.default.disable_ipv6=0
|
|
devices:
|
|
- "/dev/net/tun:/dev/net/tun"
|
|
depends_on:
|
|
client-router:
|
|
condition: "service_healthy"
|
|
api:
|
|
condition: "service_healthy"
|
|
networks:
|
|
client-internal:
|
|
ipv4_address: 172.30.0.100
|
|
ipv6_address: 172:30:0::100
|
|
extra_hosts:
|
|
- "api:203.0.113.10"
|
|
- "api:203:0:113::10"
|
|
|
|
client-router:
|
|
extends:
|
|
file: scripts/compose/router.yml
|
|
service: router
|
|
environment:
|
|
MASQUERADE_TYPE: ${CLIENT_MASQUERADE:-}
|
|
NETWORK_LATENCY_MS: 10
|
|
networks:
|
|
client-internal:
|
|
ipv4_address: 172.30.0.254
|
|
ipv6_address: 172:30:0::254
|
|
interface_name: internal
|
|
internet:
|
|
interface_name: internet
|
|
|
|
gateway:
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "ip link | grep tun-firezone"]
|
|
<<: *health-check
|
|
environment:
|
|
FIREZONE_TOKEN: ".SFMyNTY.g2gDaANtAAAAJGM4OWJjYzhjLTkzOTItNGRhZS1hNDBkLTg4OGFlZjZkMjhlMG0AAAAkMjI3NDU2MGItZTk3Yi00NWU0LThiMzQtNjc5Yzc2MTdlOThkbQAAADhPMDJMN1VTMkozVklOT01QUjlKNklMODhRSVFQNlVPOEFRVk82VTVJUEwwVkpDMjJKR0gwPT09PW4GAAH8sImUAWIAAVGA.tAm2O9FcyF67VAF3rZdwQpeADrYOIs3S2l2K51G26OM"
|
|
RUST_LOG: ${RUST_LOG:-wire=trace,debug,flow_logs=trace}
|
|
FIREZONE_API_URL: ws://api:8081
|
|
FIREZONE_ID: 4694E56C-7643-4A15-9DF3-638E5B05F570
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
|
|
# Add static route to internet subnet via router
|
|
ip -4 route add 203.0.113.0/24 via 172.31.0.254
|
|
ip -6 route add 203:0:113::/64 via 172:31:0::254
|
|
|
|
# Disable checksum offloading so that checksums are correct when they reach the relay
|
|
apk add --no-cache ethtool
|
|
ethtool -K eth0 tx off
|
|
ethtool -K eth1 tx off
|
|
ethtool -K eth2 tx off
|
|
|
|
exec firezone-gateway
|
|
init: true
|
|
build:
|
|
target: ${DOCKER_BUILD_TARGET:-debug}
|
|
context: rust
|
|
dockerfile: Dockerfile
|
|
args:
|
|
PACKAGE: firezone-gateway
|
|
image: ${GATEWAY_IMAGE:-ghcr.io/firezone/debug/gateway}:${GATEWAY_TAG:-main}
|
|
cap_add:
|
|
- NET_ADMIN
|
|
sysctls:
|
|
- net.ipv4.ip_forward=1
|
|
- net.ipv6.conf.all.disable_ipv6=0
|
|
- net.ipv6.conf.default.disable_ipv6=0
|
|
- net.ipv6.conf.all.forwarding=1
|
|
- net.ipv6.conf.default.forwarding=1
|
|
devices:
|
|
- "/dev/net/tun:/dev/net/tun"
|
|
depends_on:
|
|
gateway-router:
|
|
condition: "service_healthy"
|
|
api:
|
|
condition: "service_healthy"
|
|
networks:
|
|
gateway-internal:
|
|
ipv4_address: 172.31.0.100
|
|
ipv6_address: 172:31:0::100
|
|
dns_resources:
|
|
resources:
|
|
extra_hosts:
|
|
- "api:203.0.113.10"
|
|
- "api:203:0:113::10"
|
|
|
|
gateway-router:
|
|
extends:
|
|
file: scripts/compose/router.yml
|
|
service: router
|
|
environment:
|
|
MASQUERADE_TYPE: ${GATEWAY_MASQUERADE:-}
|
|
NETWORK_LATENCY_MS: 10
|
|
networks:
|
|
gateway-internal:
|
|
ipv4_address: 172.31.0.254
|
|
ipv6_address: 172:31:0::254
|
|
interface_name: internal
|
|
internet:
|
|
interface_name: internet
|
|
|
|
relay-1:
|
|
extends:
|
|
file: scripts/compose/relay.yml
|
|
service: relay
|
|
environment:
|
|
PUBLIC_IP4_ADDR: 203.0.113.101
|
|
PUBLIC_IP6_ADDR: 203:0:113::101
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
|
|
# Add static route to internet subnet via router
|
|
ip -4 route add 203.0.113.0/24 via 172.29.1.254
|
|
ip -6 route add 203:0:113::/64 via 172:29:1::254
|
|
|
|
apk add --no-cache ethtool
|
|
ethtool -K eth0 tx off
|
|
|
|
firezone-relay
|
|
depends_on:
|
|
relay-1-router:
|
|
condition: "service_healthy"
|
|
networks:
|
|
relay-1-internal:
|
|
ipv4_address: 172.29.1.100
|
|
ipv6_address: 172:29:1::100
|
|
|
|
relay-1-router:
|
|
extends:
|
|
file: scripts/compose/router.yml
|
|
service: router
|
|
environment:
|
|
PORT_FORWARDS: |
|
|
3478 172.29.1.100 udp
|
|
49152-65535 172.29.1.100 udp
|
|
3478 172:29:1::100 udp
|
|
49152-65535 172:29:1::100 udp
|
|
NETWORK_LATENCY_MS: 30
|
|
networks:
|
|
relay-1-internal:
|
|
ipv4_address: 172.29.1.254
|
|
ipv6_address: 172:29:1::254
|
|
interface_name: internal
|
|
internet:
|
|
ipv4_address: 203.0.113.101
|
|
ipv6_address: 203:0:113::101
|
|
interface_name: internet
|
|
|
|
relay-2:
|
|
extends:
|
|
file: scripts/compose/relay.yml
|
|
service: relay
|
|
environment:
|
|
PUBLIC_IP4_ADDR: 203.0.113.102
|
|
PUBLIC_IP6_ADDR: 203:0:113::102
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
|
|
# Add static route to internet subnet via router
|
|
ip -4 route add 203.0.113.0/24 via 172.29.2.254
|
|
ip -6 route add 203:0:113::/64 via 172:29:2::254
|
|
|
|
apk add --no-cache ethtool
|
|
ethtool -K eth0 tx off
|
|
|
|
firezone-relay
|
|
depends_on:
|
|
relay-2-router:
|
|
condition: "service_healthy"
|
|
networks:
|
|
relay-2-internal:
|
|
ipv4_address: 172.29.2.100
|
|
ipv6_address: 172:29:2::100
|
|
|
|
relay-2-router:
|
|
extends:
|
|
file: scripts/compose/router.yml
|
|
service: router
|
|
environment:
|
|
PORT_FORWARDS: |
|
|
3478 172.29.2.100 udp
|
|
49152-65535 172.29.2.100 udp
|
|
3478 172:29:2::100 udp
|
|
49152-65535 172:29:2::100 udp
|
|
NETWORK_LATENCY_MS: 30
|
|
networks:
|
|
relay-2-internal:
|
|
ipv4_address: 172.29.2.254
|
|
ipv6_address: 172:29:2::254
|
|
interface_name: internal
|
|
internet:
|
|
ipv4_address: 203.0.113.102
|
|
ipv6_address: 203:0:113::102
|
|
interface_name: internet
|
|
|
|
# The veth driver uses a pair of interfaces to connect the docker bridge to the container namespace.
|
|
# For containers that have an eBPF program attached and do XDP_TX, we need to attach a dummy program
|
|
# to the corresponding veth interface on the host to be able to receive the XDP_TX traffic and pass
|
|
# it up to the docker bridge successfully.
|
|
#
|
|
# The "recommended" way to do this is to set both veth interfaces' GRO to on, or attach an XDP program
|
|
# that does XDP_PASS to the host side veth interface. The GRO method is not reliable and was shown to
|
|
# only pass packets in large bursts every 15-20 seconds which breaks ICE setup, so we use the XDP method.
|
|
#
|
|
# For correct behaviour, we also disable any kind of offloading for all veth and bridge devices.
|
|
# This forces the kernel to calculate all checksums in software.
|
|
network-config:
|
|
image: ghcr.io/firezone/xdp-pass
|
|
pid: host
|
|
network_mode: host
|
|
privileged: true
|
|
restart: on-failure
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
|
|
VETHS=$$(ip -json link show type veth | jq -r '.[].ifname')
|
|
|
|
for dev in $$VETHS; do
|
|
echo "Attaching XDP to: $$dev"
|
|
ip link set dev $$dev xdpdrv off # Clear any existing XDP program.
|
|
ip link set dev $$dev xdpdrv obj /xdp/xdp_pass.o sec xdp
|
|
|
|
ethtool -K $$dev tx off # Disable offloading.
|
|
done
|
|
|
|
echo "Done configuring $$(echo "$$VETHS" | wc -w) veth interfaces"
|
|
|
|
BRIDGES=$$(ip -json link show type bridge | jq -r '.[].ifname')
|
|
|
|
for dev in $$BRIDGES; do
|
|
ethtool -K $$dev tx off # Disable offloading.
|
|
done
|
|
|
|
echo "Done configuring $$(echo "$$BRIDGES" | wc -w) bridge interfaces"
|
|
depends_on:
|
|
relay-1:
|
|
condition: "service_healthy"
|
|
relay-2:
|
|
condition: "service_healthy"
|
|
relay-1-router:
|
|
condition: "service_healthy"
|
|
relay-2-router:
|
|
condition: "service_healthy"
|
|
gateway-router:
|
|
condition: "service_healthy"
|
|
client-router:
|
|
condition: "service_healthy"
|
|
gateway:
|
|
condition: "service_healthy"
|
|
client:
|
|
condition: "service_healthy"
|
|
|
|
otel:
|
|
image: otel/opentelemetry-collector:latest
|
|
networks:
|
|
app-internal:
|
|
|
|
networks:
|
|
# Internet network - where all public IPs live
|
|
internet:
|
|
enable_ipv6: true
|
|
ipam:
|
|
config:
|
|
- subnet: 203.0.113.0/24
|
|
- subnet: 203:0:113::/64
|
|
|
|
app-internal:
|
|
enable_ipv6: true
|
|
ipam:
|
|
config:
|
|
- subnet: 172.28.0.0/24
|
|
- subnet: 172:28:0::/64
|
|
|
|
relay-1-internal:
|
|
enable_ipv6: true
|
|
ipam:
|
|
config:
|
|
- subnet: 172.29.1.0/24
|
|
- subnet: 172:29:1::/64
|
|
|
|
relay-2-internal:
|
|
enable_ipv6: true
|
|
ipam:
|
|
config:
|
|
- subnet: 172.29.2.0/24
|
|
- subnet: 172:29:2::/64
|
|
|
|
client-internal:
|
|
enable_ipv6: true
|
|
ipam:
|
|
config:
|
|
- subnet: 172.30.0.0/24
|
|
- subnet: 172:30:0::/64
|
|
|
|
gateway-internal:
|
|
enable_ipv6: true
|
|
ipam:
|
|
config:
|
|
- subnet: 172.31.0.0/24
|
|
- subnet: 172:31:0::/64
|