chore(ci): Add portal and relay downtime DNS resource tests (#4517)

Tests that DNS still works in the client with established connections
after the portal and/or relay go down.

.github/workflows/_integration_tests.yml

@@ -97,23 +97,23 @@ jobs:
       fail-fast: false
       matrix:
         test: [
-          direct-curl-portal-restart,
-          relayed-curl-portal-restart,
-          relayed-curl-relay-restart,
-          direct-curl-portal-down,
-          relayed-curl-portal-down,
-          direct-curl-portal-relay-down,
+          direct-curl-api-down,
+          direct-curl-api-relay-down,
+          direct-curl-api-restart,
+          direct-dns-api-down,
+          direct-dns-relay-down,
+          direct-dns,
           direct-download-roaming-network,
-          dns-etc-resolvconf,
-          dns-nm,
           dns-failsafe, # Uses the default DNS control method
+          dns-nm,
+          relayed-curl-api-down,
+          relayed-curl-api-restart,
+          relayed-curl-relay-restart,
+          relayed-dns-api-down,
+          relayed-dns-relay-restart,
+          relayed-dns,
           systemd/dns-systemd-resolved,
         ]
-        include:
-          - test: direct-download-roaming-network
-            dns-control: etc-resolv-conf
-          - test: dns-etc-resolvconf
-            dns-control: etc-resolv-conf
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/gcp-docker-login
@@ -124,16 +124,12 @@ jobs:
         run: docker compose run elixir /bin/sh -c 'cd apps/domain && mix ecto.seed'
       - name: Start docker compose in the background
         run: |
-          # Overrides the Client's env var in docker-compose.yml
-          echo "FIREZONE_DNS_CONTROL=${{ matrix.dns-control }}" >> .env
-
           # Start one-by-one to avoid variability in service startup order
           docker compose up -d dns.httpbin httpbin download.httpbin
           docker compose up -d api web domain --no-build
           docker compose up -d relay --no-build
           docker compose up -d gateway --no-build
           docker compose up -d client --no-build
-          docker compose exec -it client env
 
       - run: ./scripts/tests/${{ matrix.test }}.sh
 

docker-compose.yml

@@ -283,7 +283,7 @@ services:
 
   client:
     environment:
-      FIREZONE_DNS_CONTROL: "${FIREZONE_DNS_CONTROL}"
+      FIREZONE_DNS_CONTROL: "${FIREZONE_DNS_CONTROL:-etc-resolv-conf}"
       FIREZONE_TOKEN: "n.SFMyNTY.g2gDaANtAAAAJGM4OWJjYzhjLTkzOTItNGRhZS1hNDBkLTg4OGFlZjZkMjhlMG0AAAAkN2RhN2QxY2QtMTExYy00NGE3LWI1YWMtNDAyN2I5ZDIzMGU1bQAAACtBaUl5XzZwQmstV0xlUkFQenprQ0ZYTnFJWktXQnMyRGR3XzJ2Z0lRdkZnbgYAGUmu74wBYgABUYA.UN3vSLLcAMkHeEh5VHumPOutkuue8JA6wlxM9JxJEPE"
       RUST_LOG: firezone_linux_client=trace,wire=trace,connlib_client_shared=trace,firezone_tunnel=trace,connlib_shared=trace,boringtun=debug,snownet=debug,str0m=debug,info
       FIREZONE_API_URL: ws://api:8081

scripts/tests/direct-curl-api-down.sh

@@ -4,8 +4,8 @@ set -euo pipefail
 
 source "./scripts/tests/lib.sh"
 
-client_curl_resource # Establish a connection
+client_curl_resource "172.20.0.100/get"
 
 docker compose stop api # Stop portal
 
-client_curl_resource
+client_curl_resource "172.20.0.100/get"

scripts/tests/direct-curl-api-relay-down.sh

@@ -4,8 +4,8 @@ set -euo pipefail
 
 source "./scripts/tests/lib.sh"
 
-client_curl_resource # Establish a connection
+client_curl_resource "172.20.0.100/get"
 
 docker compose stop api relay # Stop portal & relay
 
-client_curl_resource
+client_curl_resource "172.20.0.100/get"

scripts/tests/direct-curl-api-restart.sh

@@ -6,8 +6,8 @@ source "./scripts/tests/lib.sh"
 
 docker compose restart api # Restart portal
 
-client_curl_resource
+client_curl_resource "172.20.0.100/get"
 
 docker compose restart api # Restart again
 
-client_curl_resource
+client_curl_resource "172.20.0.100/get"

scripts/tests/direct-dns-api-down.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "./scripts/tests/lib.sh"
+
+HTTPBIN=dns.httpbin
+
+function run_test() {
+    echo "# Access httpbin by DNS"
+    client_curl_resource "$HTTPBIN/get"
+
+    echo "# Make sure it's going through the tunnel"
+    client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\."
+}
+
+run_test
+
+docker compose stop api
+
+run_test

scripts/tests/direct-dns-relay-down.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "./scripts/tests/lib.sh"
+
+HTTPBIN=dns.httpbin
+
+function run_test() {
+    echo "# Access httpbin by DNS"
+    client_curl_resource "$HTTPBIN/get"
+
+    echo "# Make sure it's going through the tunnel"
+    client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\."
+}
+
+run_test
+
+docker compose stop relay
+
+run_test

scripts/tests/direct-dns.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# The integration tests call this to test Linux DNS control, using the `/etc/resolv.conf`
+# method which only works well inside Alpine Docker containers.
+
+set -euo pipefail
+
+source "./scripts/tests/lib.sh"
+
+HTTPBIN=dns.httpbin
+
+# Re-up the gateway since a local dev setup may run this back-to-back
+docker compose up -d gateway --no-build
+
+echo "# check original resolv.conf"
+client sh -c "cat /etc/resolv.conf.before-firezone"
+
+echo "# Make sure gateway can reach httpbin by DNS"
+gateway sh -c "curl --fail $HTTPBIN/get"
+
+echo "# Try to ping httpbin as a DNS resource"
+client_ping_resource "$HTTPBIN"
+
+echo "# Access httpbin by DNS"
+client_curl_resource "$HTTPBIN/get"
+
+echo "# Make sure it's going through the tunnel"
+client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\."
+
+echo "# Make sure a non-resource doesn't go through the tunnel"
+(client_nslookup "github.com" | grep "100\\.96.\\0\\.") && exit 1
+
+echo "# Stop the gateway and make sure the resource is inaccessible"
+docker compose stop gateway
+client_curl_resource "$HTTPBIN/get" && exit 1
+
+exit 0

scripts/tests/dns-failsafe.sh

@@ -7,20 +7,6 @@ set -euo pipefail
 
 source "./scripts/tests/lib.sh"
 
-function client() {
-    docker compose exec -it client "$@"
-}
-
-function client_nslookup() {
-    # Skip the first 3 lines so that grep won't see the DNS server IP
-    # `tee` here copies stdout to stderr
-    client timeout 30 sh -c "nslookup $1 | tee >(cat 1>&2) | tail -n +4"
-}
-
-function gateway() {
-    docker compose exec -it gateway "$@"
-}
-
 # Re-up the gateway since a local dev setup may run this back-to-back
 docker compose up -d gateway --no-build
 
@@ -31,11 +17,11 @@ echo "# Make sure gateway can reach httpbin by DNS"
 gateway sh -c "curl --fail dns.httpbin/get"
 
 echo "# Access httpbin by IP"
-client_curl_resource
+client_curl_resource "172.20.0.100/get"
 
 echo "# Stop the gateway and make sure the resource is inaccessible"
 docker compose stop gateway
-client sh -c "curl --connect-timeout 15 --fail 172.20.0.100/get" && exit 1
+client_curl_resource "172.20.0.100/get" && exit 1
 
 # Needed so that the previous failure doesn't bail out of the whole script
 exit 0

scripts/tests/lib.sh

@@ -1,8 +1,17 @@
 #!/usr/bin/env bash
 
+function client() {
+    docker compose exec -it client "$@"
+}
+
+function gateway() {
+    docker compose exec -it gateway "$@"
+}
+
 function install_iptables_drop_rules() {
     sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP
     sudo iptables -I FORWARD 1 -s 172.28.0.105 -d 172.28.0.100 -j DROP
+    trap remove_iptables_drop_rules EXIT # Cleanup after us
 }
 
 function remove_iptables_drop_rules() {
@@ -11,5 +20,16 @@ function remove_iptables_drop_rules() {
 }
 
 function client_curl_resource() {
-    docker compose exec -it client curl --max-time 30 --fail -i 172.20.0.100
+    client curl --fail "$1"
+}
+
+function client_ping_resource() {
+    client timeout 30 \
+        sh -c "until ping -W 1 -c 1 $1 &>/dev/null; do true; done"
+}
+
+function client_nslookup() {
+    # Skip the first 3 lines so that grep won't see the DNS server IP
+    # `tee` here copies stdout to stderr
+    client timeout 30 sh -c "nslookup $1 | tee >(cat 1>&2) | tail -n +4"
 }

scripts/tests/relayed-curl-api-down.sh

@@ -5,10 +5,9 @@ set -euo pipefail
 source "./scripts/tests/lib.sh"
 
 install_iptables_drop_rules
-trap remove_iptables_drop_rules EXIT # Cleanup after us
 
-client_curl_resource # Establish a connection
+client_curl_resource "172.20.0.100/get"
 
 docker compose stop api # Stop portal
 
-client_curl_resource
+client_curl_resource "172.20.0.100/get"

scripts/tests/relayed-curl-api-restart.sh

@@ -5,12 +5,11 @@ set -euo pipefail
 source "./scripts/tests/lib.sh"
 
 install_iptables_drop_rules
-trap remove_iptables_drop_rules EXIT # Cleanup after us
 
 docker compose restart api # Restart portal
 
-client_curl_resource
+client_curl_resource "172.20.0.100/get"
 
 docker compose restart api # Restart again
 
-client_curl_resource
+client_curl_resource "172.20.0.100/get"

scripts/tests/relayed-curl-relay-restart.sh

@@ -5,10 +5,9 @@ set -e
 source "./scripts/tests/lib.sh"
 
 install_iptables_drop_rules
-trap remove_iptables_drop_rules EXIT # Cleanup after us
 
-client_curl_resource
+client_curl_resource "172.20.0.100/get"
 
 docker compose restart relay # Restart relay
 
-client_curl_resource
+client_curl_resource "172.20.0.100/get"

scripts/tests/relayed-dns-api-down.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "./scripts/tests/lib.sh"
+
+HTTPBIN=dns.httpbin
+
+function run_test() {
+    echo "# Access httpbin by DNS"
+    client_curl_resource "$HTTPBIN/get"
+
+    echo "# Make sure it's going through the tunnel"
+    client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\."
+}
+
+install_iptables_drop_rules
+
+run_test
+
+docker compose stop api
+
+run_test

scripts/tests/relayed-dns-relay-restart.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "./scripts/tests/lib.sh"
+
+HTTPBIN=dns.httpbin
+
+function run_test() {
+    echo "# Access httpbin by DNS"
+    client_curl_resource "$HTTPBIN/get"
+
+    echo "# Make sure it's going through the tunnel"
+    client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\."
+}
+
+install_iptables_drop_rules
+
+docker compose restart relay
+
+run_test
+
+docker compose restart relay
+
+run_test

scripts/tests/relayed-dns.sh

@@ -5,21 +5,11 @@
 
 set -euo pipefail
 
+source "./scripts/tests/lib.sh"
+
 HTTPBIN=dns.httpbin
 
-function client() {
-    docker compose exec -it client "$@"
-}
-
-function client_nslookup() {
-    # Skip the first 3 lines so that grep won't see the DNS server IP
-    # `tee` here copies stdout to stderr
-    client timeout 30 sh -c "nslookup $1 | tee >(cat 1>&2) | tail -n +4"
-}
-
-function gateway() {
-    docker compose exec -it gateway "$@"
-}
+install_iptables_drop_rules
 
 # Re-up the gateway since a local dev setup may run this back-to-back
 docker compose up -d gateway --no-build
@@ -31,10 +21,10 @@ echo "# Make sure gateway can reach httpbin by DNS"
 gateway sh -c "curl --fail $HTTPBIN/get"
 
 echo "# Try to ping httpbin as a DNS resource"
-client sh -c "ping -W 1 -c 30 $HTTPBIN"
+client_ping_resource "$HTTPBIN"
 
 echo "# Access httpbin by DNS"
-client sh -c "curl --fail $HTTPBIN/get"
+client_curl_resource "$HTTPBIN/get"
 
 echo "# Make sure it's going through the tunnel"
 client_nslookup "$HTTPBIN" | grep "100\\.96\\.0\\."
@@ -44,6 +34,6 @@ echo "# Make sure a non-resource doesn't go through the tunnel"
 
 echo "# Stop the gateway and make sure the resource is inaccessible"
 docker compose stop gateway
-client sh -c "curl --connect-timeout 15 --fail $HTTPBIN/get" && exit 1
+client_curl_resource "$HTTPBIN/get" && exit 1
 
 exit 0

scripts/tests/systemd/dns-systemd-resolved.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 
 BINARY_NAME=firezone-linux-client
 
-docker compose exec client cat firezone-linux-client > "$BINARY_NAME"
+docker compose exec client cat firezone-linux-client >"$BINARY_NAME"
 chmod u+x "$BINARY_NAME"
 sudo mv "$BINARY_NAME" "/usr/bin/$BINARY_NAME"
 # TODO: Check whether this is redundant with the systemd service file
@@ -27,8 +27,7 @@ curl --interface "$FZ_IFACE" $HTTPBIN/get && exit 1
 
 echo "# Start Firezone"
 resolvectl dns tun-firezone && exit 1
-if ! sudo systemctl start firezone-client
-then
+if ! sudo systemctl start firezone-client; then
     sudo systemctl status firezone-client
     exit 1
 fi