From 09532ea845f7928a1acea4a29bfecea08f5449ed Mon Sep 17 00:00:00 2001 From: Jamil Date: Mon, 8 Apr 2024 02:43:59 -0700 Subject: [PATCH] chore(ci): Add portal and relay downtime DNS resource tests (#4517) Tests that DNS still works in the client with established connections after the portal and/or relay go down. --- .github/workflows/_integration_tests.yml | 30 +++++++-------- docker-compose.yml | 2 +- ...-relay-down.sh => direct-curl-api-down.sh} | 4 +- ...-down.sh => direct-curl-api-relay-down.sh} | 4 +- ...-restart.sh => direct-curl-api-restart.sh} | 4 +- scripts/tests/direct-dns-api-down.sh | 21 +++++++++++ scripts/tests/direct-dns-relay-down.sh | 21 +++++++++++ scripts/tests/direct-dns.sh | 37 +++++++++++++++++++ scripts/tests/dns-failsafe.sh | 18 +-------- scripts/tests/lib.sh | 22 ++++++++++- ...ortal-down.sh => relayed-curl-api-down.sh} | 5 +-- ...restart.sh => relayed-curl-api-restart.sh} | 5 +-- scripts/tests/relayed-curl-relay-restart.sh | 5 +-- scripts/tests/relayed-dns-api-down.sh | 23 ++++++++++++ scripts/tests/relayed-dns-relay-restart.sh | 25 +++++++++++++ .../{dns-etc-resolvconf.sh => relayed-dns.sh} | 22 +++-------- scripts/tests/systemd/dns-systemd-resolved.sh | 5 +-- 17 files changed, 184 insertions(+), 69 deletions(-) rename scripts/tests/{direct-curl-portal-relay-down.sh => direct-curl-api-down.sh} (58%) rename scripts/tests/{direct-curl-portal-down.sh => direct-curl-api-relay-down.sh} (61%) rename scripts/tests/{direct-curl-portal-restart.sh => direct-curl-api-restart.sh} (67%) create mode 100755 scripts/tests/direct-dns-api-down.sh create mode 100755 scripts/tests/direct-dns-relay-down.sh create mode 100755 scripts/tests/direct-dns.sh rename scripts/tests/{relayed-curl-portal-down.sh => relayed-curl-api-down.sh} (53%) rename scripts/tests/{relayed-curl-portal-restart.sh => relayed-curl-api-restart.sh} (66%) create mode 100755 scripts/tests/relayed-dns-api-down.sh create mode 100755 scripts/tests/relayed-dns-relay-restart.sh rename scripts/tests/{dns-etc-resolvconf.sh => relayed-dns.sh} (65%) diff --git a/.github/workflows/_integration_tests.yml b/.github/workflows/_integration_tests.yml index b848a4bbc..61117a216 100644 --- a/.github/workflows/_integration_tests.yml +++ b/.github/workflows/_integration_tests.yml @@ -97,23 +97,23 @@ jobs: fail-fast: false matrix: test: [ - direct-curl-portal-restart, - relayed-curl-portal-restart, - relayed-curl-relay-restart, - direct-curl-portal-down, - relayed-curl-portal-down, - direct-curl-portal-relay-down, + direct-curl-api-down, + direct-curl-api-relay-down, + direct-curl-api-restart, + direct-dns-api-down, + direct-dns-relay-down, + direct-dns, direct-download-roaming-network, - dns-etc-resolvconf, - dns-nm, dns-failsafe, # Uses the default DNS control method + dns-nm, + relayed-curl-api-down, + relayed-curl-api-restart, + relayed-curl-relay-restart, + relayed-dns-api-down, + relayed-dns-relay-restart, + relayed-dns, systemd/dns-systemd-resolved, ] - include: - - test: direct-download-roaming-network - dns-control: etc-resolv-conf - - test: dns-etc-resolvconf - dns-control: etc-resolv-conf steps: - uses: actions/checkout@v4 - uses: ./.github/actions/gcp-docker-login @@ -124,16 +124,12 @@ jobs: run: docker compose run elixir /bin/sh -c 'cd apps/domain && mix ecto.seed' - name: Start docker compose in the background run: | - # Overrides the Client's env var in docker-compose.yml - echo "FIREZONE_DNS_CONTROL=${{ matrix.dns-control }}" >> .env - # Start one-by-one to avoid variability in service startup order docker compose up -d dns.httpbin httpbin download.httpbin docker compose up -d api web domain --no-build docker compose up -d relay --no-build docker compose up -d gateway --no-build docker compose up -d client --no-build - docker compose exec -it client env - run: ./scripts/tests/${{ matrix.test }}.sh diff --git a/docker-compose.yml b/docker-compose.yml index b3dbb4fa9..7852c70e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -283,7 +283,7 @@ services: client: environment: - FIREZONE_DNS_CONTROL: "${FIREZONE_DNS_CONTROL}" + FIREZONE_DNS_CONTROL: "${FIREZONE_DNS_CONTROL:-etc-resolv-conf}" FIREZONE_TOKEN: "n.SFMyNTY.g2gDaANtAAAAJGM4OWJjYzhjLTkzOTItNGRhZS1hNDBkLTg4OGFlZjZkMjhlMG0AAAAkN2RhN2QxY2QtMTExYy00NGE3LWI1YWMtNDAyN2I5ZDIzMGU1bQAAACtBaUl5XzZwQmstV0xlUkFQenprQ0ZYTnFJWktXQnMyRGR3XzJ2Z0lRdkZnbgYAGUmu74wBYgABUYA.UN3vSLLcAMkHeEh5VHumPOutkuue8JA6wlxM9JxJEPE" RUST_LOG: firezone_linux_client=trace,wire=trace,connlib_client_shared=trace,firezone_tunnel=trace,connlib_shared=trace,boringtun=debug,snownet=debug,str0m=debug,info FIREZONE_API_URL: ws://api:8081 diff --git a/scripts/tests/direct-curl-portal-relay-down.sh b/scripts/tests/direct-curl-api-down.sh similarity index 58% rename from scripts/tests/direct-curl-portal-relay-down.sh rename to scripts/tests/direct-curl-api-down.sh index e75eb3e82..72b4aa051 100755 --- a/scripts/tests/direct-curl-portal-relay-down.sh +++ b/scripts/tests/direct-curl-api-down.sh @@ -4,8 +4,8 @@ set -euo pipefail source "./scripts/tests/lib.sh" -client_curl_resource # Establish a connection +client_curl_resource "172.20.0.100/get" docker compose stop api # Stop portal -client_curl_resource +client_curl_resource "172.20.0.100/get" diff --git a/scripts/tests/direct-curl-portal-down.sh b/scripts/tests/direct-curl-api-relay-down.sh similarity index 61% rename from scripts/tests/direct-curl-portal-down.sh rename to scripts/tests/direct-curl-api-relay-down.sh index f21a26cd2..e128e18dd 100755 --- a/scripts/tests/direct-curl-portal-down.sh +++ b/scripts/tests/direct-curl-api-relay-down.sh @@ -4,8 +4,8 @@ set -euo pipefail source "./scripts/tests/lib.sh" -client_curl_resource # Establish a connection +client_curl_resource "172.20.0.100/get" docker compose stop api relay # Stop portal & relay -client_curl_resource +client_curl_resource "172.20.0.100/get" diff --git a/scripts/tests/direct-curl-portal-restart.sh b/scripts/tests/direct-curl-api-restart.sh similarity index 67% rename from scripts/tests/direct-curl-portal-restart.sh rename to scripts/tests/direct-curl-api-restart.sh index 8c6d42bb9..d17095eb0 100755 --- a/scripts/tests/direct-curl-portal-restart.sh +++ b/scripts/tests/direct-curl-api-restart.sh @@ -6,8 +6,8 @@ source "./scripts/tests/lib.sh" docker compose restart api # Restart portal -client_curl_resource +client_curl_resource "172.20.0.100/get" docker compose restart api # Restart again -client_curl_resource +client_curl_resource "172.20.0.100/get" diff --git a/scripts/tests/direct-dns-api-down.sh b/scripts/tests/direct-dns-api-down.sh new file mode 100755 index 000000000..13b6e6983 --- /dev/null +++ b/scripts/tests/direct-dns-api-down.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "./scripts/tests/lib.sh" + +HTTPBIN=dns.httpbin + +function run_test() { + echo "# Access httpbin by DNS" + client_curl_resource "$HTTPBIN/get" + + echo "# Make sure it's going through the tunnel" + client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\." +} + +run_test + +docker compose stop api + +run_test diff --git a/scripts/tests/direct-dns-relay-down.sh b/scripts/tests/direct-dns-relay-down.sh new file mode 100755 index 000000000..be57f9c65 --- /dev/null +++ b/scripts/tests/direct-dns-relay-down.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "./scripts/tests/lib.sh" + +HTTPBIN=dns.httpbin + +function run_test() { + echo "# Access httpbin by DNS" + client_curl_resource "$HTTPBIN/get" + + echo "# Make sure it's going through the tunnel" + client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\." +} + +run_test + +docker compose stop relay + +run_test diff --git a/scripts/tests/direct-dns.sh b/scripts/tests/direct-dns.sh new file mode 100755 index 000000000..883a620ae --- /dev/null +++ b/scripts/tests/direct-dns.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# The integration tests call this to test Linux DNS control, using the `/etc/resolv.conf` +# method which only works well inside Alpine Docker containers. + +set -euo pipefail + +source "./scripts/tests/lib.sh" + +HTTPBIN=dns.httpbin + +# Re-up the gateway since a local dev setup may run this back-to-back +docker compose up -d gateway --no-build + +echo "# check original resolv.conf" +client sh -c "cat /etc/resolv.conf.before-firezone" + +echo "# Make sure gateway can reach httpbin by DNS" +gateway sh -c "curl --fail $HTTPBIN/get" + +echo "# Try to ping httpbin as a DNS resource" +client_ping_resource "$HTTPBIN" + +echo "# Access httpbin by DNS" +client_curl_resource "$HTTPBIN/get" + +echo "# Make sure it's going through the tunnel" +client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\." + +echo "# Make sure a non-resource doesn't go through the tunnel" +(client_nslookup "github.com" | grep "100\\.96.\\0\\.") && exit 1 + +echo "# Stop the gateway and make sure the resource is inaccessible" +docker compose stop gateway +client_curl_resource "$HTTPBIN/get" && exit 1 + +exit 0 diff --git a/scripts/tests/dns-failsafe.sh b/scripts/tests/dns-failsafe.sh index 20f26a064..2b3bc3859 100755 --- a/scripts/tests/dns-failsafe.sh +++ b/scripts/tests/dns-failsafe.sh @@ -7,20 +7,6 @@ set -euo pipefail source "./scripts/tests/lib.sh" -function client() { - docker compose exec -it client "$@" -} - -function client_nslookup() { - # Skip the first 3 lines so that grep won't see the DNS server IP - # `tee` here copies stdout to stderr - client timeout 30 sh -c "nslookup $1 | tee >(cat 1>&2) | tail -n +4" -} - -function gateway() { - docker compose exec -it gateway "$@" -} - # Re-up the gateway since a local dev setup may run this back-to-back docker compose up -d gateway --no-build @@ -31,11 +17,11 @@ echo "# Make sure gateway can reach httpbin by DNS" gateway sh -c "curl --fail dns.httpbin/get" echo "# Access httpbin by IP" -client_curl_resource +client_curl_resource "172.20.0.100/get" echo "# Stop the gateway and make sure the resource is inaccessible" docker compose stop gateway -client sh -c "curl --connect-timeout 15 --fail 172.20.0.100/get" && exit 1 +client_curl_resource "172.20.0.100/get" && exit 1 # Needed so that the previous failure doesn't bail out of the whole script exit 0 diff --git a/scripts/tests/lib.sh b/scripts/tests/lib.sh index 56cf7eb25..7c3a472f7 100755 --- a/scripts/tests/lib.sh +++ b/scripts/tests/lib.sh @@ -1,8 +1,17 @@ #!/usr/bin/env bash +function client() { + docker compose exec -it client "$@" +} + +function gateway() { + docker compose exec -it gateway "$@" +} + function install_iptables_drop_rules() { sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP sudo iptables -I FORWARD 1 -s 172.28.0.105 -d 172.28.0.100 -j DROP + trap remove_iptables_drop_rules EXIT # Cleanup after us } function remove_iptables_drop_rules() { @@ -11,5 +20,16 @@ function remove_iptables_drop_rules() { } function client_curl_resource() { - docker compose exec -it client curl --max-time 30 --fail -i 172.20.0.100 + client curl --fail "$1" +} + +function client_ping_resource() { + client timeout 30 \ + sh -c "until ping -W 1 -c 1 $1 &>/dev/null; do true; done" +} + +function client_nslookup() { + # Skip the first 3 lines so that grep won't see the DNS server IP + # `tee` here copies stdout to stderr + client timeout 30 sh -c "nslookup $1 | tee >(cat 1>&2) | tail -n +4" } diff --git a/scripts/tests/relayed-curl-portal-down.sh b/scripts/tests/relayed-curl-api-down.sh similarity index 53% rename from scripts/tests/relayed-curl-portal-down.sh rename to scripts/tests/relayed-curl-api-down.sh index 2114aa4ad..5ce0ce907 100755 --- a/scripts/tests/relayed-curl-portal-down.sh +++ b/scripts/tests/relayed-curl-api-down.sh @@ -5,10 +5,9 @@ set -euo pipefail source "./scripts/tests/lib.sh" install_iptables_drop_rules -trap remove_iptables_drop_rules EXIT # Cleanup after us -client_curl_resource # Establish a connection +client_curl_resource "172.20.0.100/get" docker compose stop api # Stop portal -client_curl_resource +client_curl_resource "172.20.0.100/get" diff --git a/scripts/tests/relayed-curl-portal-restart.sh b/scripts/tests/relayed-curl-api-restart.sh similarity index 66% rename from scripts/tests/relayed-curl-portal-restart.sh rename to scripts/tests/relayed-curl-api-restart.sh index e81dc2cbd..b152cd80e 100755 --- a/scripts/tests/relayed-curl-portal-restart.sh +++ b/scripts/tests/relayed-curl-api-restart.sh @@ -5,12 +5,11 @@ set -euo pipefail source "./scripts/tests/lib.sh" install_iptables_drop_rules -trap remove_iptables_drop_rules EXIT # Cleanup after us docker compose restart api # Restart portal -client_curl_resource +client_curl_resource "172.20.0.100/get" docker compose restart api # Restart again -client_curl_resource +client_curl_resource "172.20.0.100/get" diff --git a/scripts/tests/relayed-curl-relay-restart.sh b/scripts/tests/relayed-curl-relay-restart.sh index 36921bad5..45fec9782 100755 --- a/scripts/tests/relayed-curl-relay-restart.sh +++ b/scripts/tests/relayed-curl-relay-restart.sh @@ -5,10 +5,9 @@ set -e source "./scripts/tests/lib.sh" install_iptables_drop_rules -trap remove_iptables_drop_rules EXIT # Cleanup after us -client_curl_resource +client_curl_resource "172.20.0.100/get" docker compose restart relay # Restart relay -client_curl_resource +client_curl_resource "172.20.0.100/get" diff --git a/scripts/tests/relayed-dns-api-down.sh b/scripts/tests/relayed-dns-api-down.sh new file mode 100755 index 000000000..234cb0cfc --- /dev/null +++ b/scripts/tests/relayed-dns-api-down.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "./scripts/tests/lib.sh" + +HTTPBIN=dns.httpbin + +function run_test() { + echo "# Access httpbin by DNS" + client_curl_resource "$HTTPBIN/get" + + echo "# Make sure it's going through the tunnel" + client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\." +} + +install_iptables_drop_rules + +run_test + +docker compose stop api + +run_test diff --git a/scripts/tests/relayed-dns-relay-restart.sh b/scripts/tests/relayed-dns-relay-restart.sh new file mode 100755 index 000000000..45fb05211 --- /dev/null +++ b/scripts/tests/relayed-dns-relay-restart.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "./scripts/tests/lib.sh" + +HTTPBIN=dns.httpbin + +function run_test() { + echo "# Access httpbin by DNS" + client_curl_resource "$HTTPBIN/get" + + echo "# Make sure it's going through the tunnel" + client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\." +} + +install_iptables_drop_rules + +docker compose restart relay + +run_test + +docker compose restart relay + +run_test diff --git a/scripts/tests/dns-etc-resolvconf.sh b/scripts/tests/relayed-dns.sh similarity index 65% rename from scripts/tests/dns-etc-resolvconf.sh rename to scripts/tests/relayed-dns.sh index 5f5437438..325bec771 100755 --- a/scripts/tests/dns-etc-resolvconf.sh +++ b/scripts/tests/relayed-dns.sh @@ -5,21 +5,11 @@ set -euo pipefail +source "./scripts/tests/lib.sh" + HTTPBIN=dns.httpbin -function client() { - docker compose exec -it client "$@" -} - -function client_nslookup() { - # Skip the first 3 lines so that grep won't see the DNS server IP - # `tee` here copies stdout to stderr - client timeout 30 sh -c "nslookup $1 | tee >(cat 1>&2) | tail -n +4" -} - -function gateway() { - docker compose exec -it gateway "$@" -} +install_iptables_drop_rules # Re-up the gateway since a local dev setup may run this back-to-back docker compose up -d gateway --no-build @@ -31,10 +21,10 @@ echo "# Make sure gateway can reach httpbin by DNS" gateway sh -c "curl --fail $HTTPBIN/get" echo "# Try to ping httpbin as a DNS resource" -client sh -c "ping -W 1 -c 30 $HTTPBIN" +client_ping_resource "$HTTPBIN" echo "# Access httpbin by DNS" -client sh -c "curl --fail $HTTPBIN/get" +client_curl_resource "$HTTPBIN/get" echo "# Make sure it's going through the tunnel" client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\." @@ -44,6 +34,6 @@ echo "# Make sure a non-resource doesn't go through the tunnel" echo "# Stop the gateway and make sure the resource is inaccessible" docker compose stop gateway -client sh -c "curl --connect-timeout 15 --fail $HTTPBIN/get" && exit 1 +client_curl_resource "$HTTPBIN/get" && exit 1 exit 0 diff --git a/scripts/tests/systemd/dns-systemd-resolved.sh b/scripts/tests/systemd/dns-systemd-resolved.sh index 0d0fb5f92..acf49523a 100755 --- a/scripts/tests/systemd/dns-systemd-resolved.sh +++ b/scripts/tests/systemd/dns-systemd-resolved.sh @@ -5,7 +5,7 @@ set -euo pipefail BINARY_NAME=firezone-linux-client -docker compose exec client cat firezone-linux-client > "$BINARY_NAME" +docker compose exec client cat firezone-linux-client >"$BINARY_NAME" chmod u+x "$BINARY_NAME" sudo mv "$BINARY_NAME" "/usr/bin/$BINARY_NAME" # TODO: Check whether this is redundant with the systemd service file @@ -27,8 +27,7 @@ curl --interface "$FZ_IFACE" $HTTPBIN/get && exit 1 echo "# Start Firezone" resolvectl dns tun-firezone && exit 1 -if ! sudo systemctl start firezone-client -then +if ! sudo systemctl start firezone-client; then sudo systemctl status firezone-client exit 1 fi