mirror of
https://github.com/outbackdingo/firezone.git
synced 2026-01-27 10:18:54 +00:00
ci: create a more realistic network setup (#10301)
Currently, the setup we have in docker-compose does not reflect real-world scenarios very well because most components share the same subnet. In reality, Clients, Gateways, relays and the backend are all in separate subnets, connected via multiple routers on the Internet. The current setup makes it hard to properly test relayed connections. To fix this, we move all components into their own subnet with a dedicated router container that performs source and destination NAT as well as acts as a firewall for the client and gateway containers to not allow inbound traffic. This setup will allow us to more easily test #10286 which requires port randomization for outgoing traffic on the Client and Gateway side.
This commit is contained in:
Thomas Eizinger
12
scripts/router/Dockerfile
Normal file
12
scripts/router/Dockerfile
Normal file
@@ -0,0 +1,12 @@
|
||||
FROM alpine:3.22
|
||||
|
||||
RUN apk add --no-cache iproute2 nftables jq bash
|
||||
|
||||
HEALTHCHECK --interval=1s --timeout=1s --retries=5 CMD [ "sh", "-c", "test $(cat /tmp/setup_done) = 1" ]
|
||||
|
||||
WORKDIR /bin
|
||||
COPY ./router.sh /bin/router.sh
|
||||
COPY ./router.nft /bin/router.nft
|
||||
RUN chmod +x /bin/router.sh
|
||||
|
||||
CMD ["router.sh"]
|
||||
15
scripts/router/README.md
Normal file
15
scripts/router/README.md
Normal file
@@ -0,0 +1,15 @@
|
||||
# Router
|
||||
|
||||
This container acts as a simple router how they are found on the public Internet.
|
||||
By default, no inbound traffic is allowed, except for:
|
||||
|
||||
- responses of previously outgoing connections
|
||||
- explicit port forwarding
|
||||
|
||||
The router uses `nftables` to enforce these rules.
|
||||
|
||||
We also make several assumptions about the docker-compose setup that we are running in:
|
||||
|
||||
- The network interface between the router and its container must be called `internal`
|
||||
- The public network interface on the other side must be called `internet`
|
||||
- IPv4 and IPv6 must be available on both interfaces
|
||||
36
scripts/router/router.nft
Normal file
36
scripts/router/router.nft
Normal file
@@ -0,0 +1,36 @@
|
||||
table inet router {
|
||||
# Input chain - drop by default, allow established connections
|
||||
chain input {
|
||||
type filter hook input priority filter; policy drop;
|
||||
|
||||
# Allow loopback
|
||||
iif "lo" accept
|
||||
|
||||
# Allow established and related connections
|
||||
ct state established,related accept
|
||||
|
||||
# Allow ICMP/ICMPv6 for basic connectivity
|
||||
ip protocol icmp accept
|
||||
ip6 nexthdr ipv6-icmp accept
|
||||
}
|
||||
|
||||
# Forward chain - accept by default for router functionality
|
||||
chain forward {
|
||||
type filter hook forward priority filter; policy accept;
|
||||
}
|
||||
|
||||
# Output chain - accept by default
|
||||
chain output {
|
||||
type filter hook output priority filter; policy accept;
|
||||
}
|
||||
|
||||
# Prerouting chain for DNAT
|
||||
chain prerouting {
|
||||
type nat hook prerouting priority dstnat;
|
||||
}
|
||||
|
||||
# Postrouting chain for SNAT/masquerading
|
||||
chain postrouting {
|
||||
type nat hook postrouting priority srcnat;
|
||||
}
|
||||
}
|
||||
78
scripts/router/router.sh
Normal file
78
scripts/router/router.sh
Normal file
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Get network configuration
|
||||
INTERNAL_NET_V4=$(ip -4 --json route | jq -r '.[] | select(.dev == "internal") | select(.dst == "default" | not) | .dst')
|
||||
INTERNAL_NET_V6=$(ip -6 --json route | jq -r '.[] | select(.dev == "internal") | select(.dst | startswith("fe80") | not) | select(.dst == "default" | not) | .dst')
|
||||
PUBLIC_IPV4=$(ip -4 -json addr show internet | jq -r '.[0].addr_info[0].local')
|
||||
PUBLIC_IPV6=$(ip -6 -json addr show internet | jq -r '.[0].addr_info[0].local')
|
||||
|
||||
# Validate required configuration
|
||||
if [ -z "$INTERNAL_NET_V4" ]; then
|
||||
echo "Error: Failed to identify internal IPv4 subnet"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$INTERNAL_NET_V6" ]; then
|
||||
echo "Error: Failed to identify internal IPv6 subnet"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$PUBLIC_IPV4" ]; then
|
||||
echo "Error: Failed to get public IPv4"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$PUBLIC_IPV6" ]; then
|
||||
echo "Error: Failed to get public IPv6"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "INTERNAL_NET_V4 = $INTERNAL_NET_V4"
|
||||
echo "INTERNAL_NET_V6 = $INTERNAL_NET_V6"
|
||||
echo "PUBLIC_IPV4 = $PUBLIC_IPV4"
|
||||
echo "PUBLIC_IPV6 = $PUBLIC_IPV6"
|
||||
|
||||
TEMPLATE_FILE="router.nft"
|
||||
CONFIG_FILE="/tmp/router.nft"
|
||||
|
||||
# Copy template file to working config
|
||||
cp "$TEMPLATE_FILE" "$CONFIG_FILE"
|
||||
|
||||
echo "add rule inet router postrouting ip saddr $INTERNAL_NET_V4 oifname \"internet\" masquerade ${MASQUERADE_TYPE:-}" >>"$CONFIG_FILE"
|
||||
echo "add rule inet router postrouting ip6 saddr $INTERNAL_NET_V6 oifname \"internet\" masquerade ${MASQUERADE_TYPE:-}" >>"$CONFIG_FILE"
|
||||
|
||||
# Add port forwarding rules if specified
|
||||
if [ -n "${PORT_FORWARDS:-}" ]; then
|
||||
echo "$PORT_FORWARDS" | tr ',' '\n' | while IFS=' ' read -r port internal_ip protocol; do
|
||||
if [ -z "$port" ] || [ -z "$internal_ip" ] || [ -z "$protocol" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Determine if internal IP is IPv4 or IPv6 and append rules to config file
|
||||
case "$internal_ip" in
|
||||
*:*) # IPv6 address
|
||||
echo "add rule inet router prerouting ip6 daddr $PUBLIC_IPV6 $protocol dport $port dnat to [$internal_ip]:$port" >>"$CONFIG_FILE"
|
||||
echo "add rule inet router input ip6 daddr $internal_ip $protocol dport $port accept" >>"$CONFIG_FILE"
|
||||
;;
|
||||
*) # IPv4 address
|
||||
echo "add rule inet router prerouting ip daddr $PUBLIC_IPV4 $protocol dport $port dnat to $internal_ip:$port" >>"$CONFIG_FILE"
|
||||
echo "add rule inet router input ip daddr $internal_ip $protocol dport $port accept" >>"$CONFIG_FILE"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
echo "-----------------------------------------------------------------------------------------------"
|
||||
cat "$CONFIG_FILE"
|
||||
echo "-----------------------------------------------------------------------------------------------"
|
||||
|
||||
nft -f "$CONFIG_FILE"
|
||||
|
||||
rm "$CONFIG_FILE"
|
||||
|
||||
echo "1" >/tmp/setup_done # Health check marker
|
||||
|
||||
# Keep container running
|
||||
exec tail -f /dev/null
|
||||
@@ -17,13 +17,13 @@ DOWNLOAD_PID=$!
|
||||
|
||||
sleep 3 # Download a bit
|
||||
|
||||
docker network disconnect firezone_app firezone-client-1 # Disconnect the client
|
||||
docker network disconnect firezone_client-internal firezone-client-1 # Disconnect the client
|
||||
sleep 3
|
||||
docker network connect firezone_app firezone-client-1 --ip 172.28.0.200 # Reconnect client with a different IP
|
||||
docker network connect firezone_client-internal firezone-client-1 --ip 172.30.0.200 --ip6 172:30::200 # Reconnect client with a different IP
|
||||
|
||||
# Re-add static route to relays through router
|
||||
client ip route add 172.29.0.0/24 via 172.28.0.254 dev eth0
|
||||
client ip -6 route add 172:29:0::/64 via 172:28:0::254 dev eth0
|
||||
# Add static route to internet subnet via router; they get removed when the network interface disappears
|
||||
client ip -4 route add 203.0.113.0/24 via 172.30.0.254
|
||||
client ip -6 route add 203:0:113::/64 via 172:30:0::254
|
||||
|
||||
# Send SIGHUP, triggering `reconnect` internally
|
||||
sudo kill -s HUP "$(ps -C firezone-headless-client -o pid=)"
|
||||
|
||||
@@ -18,55 +18,6 @@ function relay2() {
|
||||
docker compose exec -T relay-2 "$@"
|
||||
}
|
||||
|
||||
# Takes two optional arguments to force the client and gateway to use a specific IP stack.
|
||||
# 1. client_stack: "ipv4", "ipv6"
|
||||
# 2. gateway_stack: "ipv4", "ipv6"
|
||||
#
|
||||
# By default, the client and gateway will use happy eyeballs to use pick the first working IP stack.
|
||||
function force_relayed_connections() {
|
||||
# Install `iptables` to have it available in the compatibility tests
|
||||
client apk add --no-cache iptables
|
||||
|
||||
# Execute within the client container because doing so from the host is not reliable in CI.
|
||||
client iptables -A OUTPUT -d 172.28.0.105 -j DROP
|
||||
client ip6tables -A OUTPUT -d 172:28:0::105 -j DROP
|
||||
|
||||
local client_stack="${1:-}"
|
||||
local gateway_stack="${2:-}"
|
||||
|
||||
# If both are empty, we don't care which stack they use; just return
|
||||
if [[ -z "$client_stack" && -z "$gateway_stack" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
gateway apk add --no-cache iptables
|
||||
|
||||
if [[ "$client_stack" == "ipv4" && "$gateway_stack" == "ipv4" ]]; then
|
||||
client ip6tables -A OUTPUT -d $RELAY_1_PUBLIC_IP6_ADDR -j DROP
|
||||
client ip6tables -A OUTPUT -d $RELAY_2_PUBLIC_IP6_ADDR -j DROP
|
||||
gateway ip6tables -A OUTPUT -d $RELAY_1_PUBLIC_IP6_ADDR -j DROP
|
||||
gateway ip6tables -A OUTPUT -d $RELAY_2_PUBLIC_IP6_ADDR -j DROP
|
||||
elif [[ "$client_stack" == "ipv4" && "$gateway_stack" == "ipv6" ]]; then
|
||||
client ip6tables -A OUTPUT -d $RELAY_1_PUBLIC_IP6_ADDR -j DROP
|
||||
client ip6tables -A OUTPUT -d $RELAY_2_PUBLIC_IP6_ADDR -j DROP
|
||||
gateway iptables -A OUTPUT -d $RELAY_1_PUBLIC_IP4_ADDR -j DROP
|
||||
gateway iptables -A OUTPUT -d $RELAY_2_PUBLIC_IP4_ADDR -j DROP
|
||||
elif [[ "$client_stack" == "ipv6" && "$gateway_stack" == "ipv4" ]]; then
|
||||
client iptables -A OUTPUT -d $RELAY_1_PUBLIC_IP4_ADDR -j DROP
|
||||
client iptables -A OUTPUT -d $RELAY_2_PUBLIC_IP4_ADDR -j DROP
|
||||
gateway ip6tables -A OUTPUT -d $RELAY_1_PUBLIC_IP6_ADDR -j DROP
|
||||
gateway ip6tables -A OUTPUT -d $RELAY_2_PUBLIC_IP6_ADDR -j DROP
|
||||
elif [[ "$client_stack" == "ipv6" && "$gateway_stack" == "ipv6" ]]; then
|
||||
client iptables -A OUTPUT -d $RELAY_1_PUBLIC_IP4_ADDR -j DROP
|
||||
client iptables -A OUTPUT -d $RELAY_2_PUBLIC_IP4_ADDR -j DROP
|
||||
gateway iptables -A OUTPUT -d $RELAY_1_PUBLIC_IP4_ADDR -j DROP
|
||||
gateway iptables -A OUTPUT -d $RELAY_2_PUBLIC_IP4_ADDR -j DROP
|
||||
else
|
||||
echo "Invalid stack combination: client_stack=$client_stack, gateway_stack=$gateway_stack"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function client_curl_resource() {
|
||||
client curl --connect-timeout 30 --fail "$1" >/dev/null
|
||||
}
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euox pipefail
|
||||
|
||||
source "./scripts/tests/lib.sh"
|
||||
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 \
|
||||
--time 30 \
|
||||
--udp \
|
||||
--bandwidth 600M \
|
||||
--client 172.20.0.110 \
|
||||
--json' >>"${TEST_NAME}.json"
|
||||
|
||||
assert_process_state "gateway" "S"
|
||||
assert_process_state "client" "S"
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euox pipefail
|
||||
|
||||
source "./scripts/tests/lib.sh"
|
||||
force_relayed_connections ipv4 ipv4
|
||||
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 \
|
||||
--time 30 \
|
||||
--client 172.20.0.110 \
|
||||
--json' >>"${TEST_NAME}.json"
|
||||
|
||||
assert_process_state "relay-1" "S"
|
||||
assert_process_state "relay-2" "S"
|
||||
assert_process_state "gateway" "S"
|
||||
assert_process_state "client" "S"
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euox pipefail
|
||||
|
||||
source "./scripts/tests/lib.sh"
|
||||
force_relayed_connections ipv6 ipv4
|
||||
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 \
|
||||
--time 30 \
|
||||
--reverse \
|
||||
--client 172.20.0.110 \
|
||||
--json' >>"${TEST_NAME}.json"
|
||||
|
||||
assert_process_state "relay-1" "S"
|
||||
assert_process_state "relay-2" "S"
|
||||
assert_process_state "gateway" "S"
|
||||
assert_process_state "client" "S"
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euox pipefail
|
||||
|
||||
source "./scripts/tests/lib.sh"
|
||||
force_relayed_connections ipv6 ipv6
|
||||
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 \
|
||||
--time 30 \
|
||||
--udp \
|
||||
--bandwidth 300M \
|
||||
--client 172.20.0.110 \
|
||||
--json' >>"${TEST_NAME}.json"
|
||||
|
||||
assert_process_state "relay-1" "S"
|
||||
assert_process_state "relay-2" "S"
|
||||
assert_process_state "gateway" "S"
|
||||
assert_process_state "client" "S"
|
||||
@@ -1,19 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euox pipefail
|
||||
|
||||
source "./scripts/tests/lib.sh"
|
||||
force_relayed_connections ipv4 ipv6
|
||||
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 \
|
||||
--time 30 \
|
||||
--reverse \
|
||||
--udp \
|
||||
--bandwidth 300M \
|
||||
--client 172.20.0.110 \
|
||||
--json' >>"${TEST_NAME}.json"
|
||||
|
||||
assert_process_state "relay-1" "S"
|
||||
assert_process_state "relay-2" "S"
|
||||
assert_process_state "gateway" "S"
|
||||
assert_process_state "client" "S"
|
||||
15
scripts/tests/perf/udp-client2server.sh
Executable file
15
scripts/tests/perf/udp-client2server.sh
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euox pipefail
|
||||
|
||||
source "./scripts/tests/lib.sh"
|
||||
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c "iperf3 \
|
||||
--time 30 \
|
||||
--udp \
|
||||
--bandwidth ${UDP_BITRATE:-450M} \
|
||||
--client 172.20.0.110 \
|
||||
--json" >>"${TEST_NAME}.json"
|
||||
|
||||
assert_process_state "gateway" "S"
|
||||
assert_process_state "client" "S"
|
||||
@@ -4,13 +4,13 @@ set -euox pipefail
|
||||
|
||||
source "./scripts/tests/lib.sh"
|
||||
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 \
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c "iperf3 \
|
||||
--time 30 \
|
||||
--reverse \
|
||||
--udp \
|
||||
--bandwidth 600M \
|
||||
--bandwidth ${UDP_BITRATE:-450M} \
|
||||
--client 172.20.0.110 \
|
||||
--json' >>"${TEST_NAME}.json"
|
||||
--json" >>"${TEST_NAME}.json"
|
||||
|
||||
assert_process_state "gateway" "S"
|
||||
assert_process_state "client" "S"
|
||||
@@ -13,6 +13,7 @@ debug_exit() {
|
||||
docker compose ps -a
|
||||
resolvectl dns tun-firezone || true
|
||||
systemctl status "$SERVICE_NAME" || true
|
||||
journalctl -eu "$SERVICE_NAME" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -52,7 +53,7 @@ resolvectl query "$HTTPBIN" || debug_exit
|
||||
# Accessing a resource should succeed after the client is up
|
||||
# Block off Docker's DNS.
|
||||
sudo resolvectl dns "$DOCKER_IFACE" ""
|
||||
curl -v $HTTPBIN/get
|
||||
curl -v $HTTPBIN/get || debug_exit
|
||||
|
||||
# Make sure it's going through the tunnel
|
||||
nslookup "$HTTPBIN" | grep "100\\.96\\.0\\."
|
||||
|
||||
Reference in New Issue
Block a user