ci: ensure roaming between networks doesn't abort file download (#4213)

This adds an integration test that downloads a 10MB file from a server
and simulates the client roaming to another network while the download
is active.

We use a DNS resource for this to ensure it also doesn't take too long
in that case. DNS resources are what most users will be using and we
clear some internal DNS caches on connection failures. Hence, using a
DNS resource here is a somewhat roundabout way to test that we aren't
failing and re-establishing the connection but migrate it to a new
network path.

.github/workflows/_build_artifacts.yml

@@ -183,6 +183,9 @@ jobs:
           - package: snownet-tests
             artifact: snownet-tests
             image_name: snownet-tests
+          - package: http-test-server
+            artifact: http-test-server
+            image_name: http-test-server
     env:
       BINARY_DEST_PATH: ${{ matrix.name.artifact }}-${{ matrix.arch.shortname }}
     outputs:
@@ -319,6 +322,7 @@ jobs:
           - gateway
           - client
           - snownet-tests
+          - http-test-server
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/_integration_tests.yml

@@ -59,6 +59,14 @@ on:
         required: false
         type: string
         default: ${{ github.sha }}
+      http_test_server_image:
+        required: false
+        type: string
+        default: 'us-east1-docker.pkg.dev/firezone-staging/firezone/debug/http-test-server'
+      http_test_server_tag:
+        required: false
+        type: string
+        default: ${{ github.sha }}
 
 jobs:
   integration-tests:
@@ -83,6 +91,8 @@ jobs:
       CLIENT_TAG: ${{ inputs.client_tag }}
       ELIXIR_IMAGE: ${{ inputs.elixir_image }}
       ELIXIR_TAG: ${{ inputs.elixir_tag }}
+      HTTP_TEST_SERVER_IMAGE: ${{ inputs.http_test_server_image }}
+      HTTP_TEST_SERVER_TAG: ${{ inputs.http_test_server_tag }}
     strategy:
       fail-fast: false
       matrix:
@@ -93,6 +103,7 @@ jobs:
           direct-curl-portal-down,
           relayed-curl-portal-down,
           direct-curl-portal-relay-down,
+          direct-download-roaming-network,
           dns-etc-resolvconf,
           dns-nm,
           systemd/dns-systemd-resolved,
@@ -108,7 +119,7 @@ jobs:
       - name: Start docker compose in the background
         run: |
           # Start one-by-one to avoid variability in service startup order
-          docker compose up -d dns.httpbin httpbin
+          docker compose up -d dns.httpbin httpbin download.httpbin
           docker compose up -d api web domain --no-build
           docker compose up -d relay --no-build
           docker compose up -d gateway --no-build

docker-compose.yml

@@ -357,6 +357,22 @@ services:
       resources:
         ipv4_address: 172.20.0.100
 
+  download.httpbin: # Named after `httpbin` because that is how DNS resources are configured for the test setup.
+    build:
+      target: dev
+      context: rust
+      dockerfile: Dockerfile
+      cache_from:
+        - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/http-test-server:main
+      args:
+        PACKAGE: http-test-server
+    image: ${HTTP_TEST_SERVER_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/dev/http-test-server}:${HTTP_TEST_SERVER_TAG:-main}
+    environment:
+      PORT: 80
+    networks:
+      dns_resources:
+        ipv4_address: 172.21.0.101
+
   dns.httpbin:
     image: kennethreitz/httpbin
     healthcheck:

rust/Cargo.lock

@@ -489,11 +489,15 @@ dependencies = [
  "pin-project-lite",
  "rustversion",
  "serde",
+ "serde_json",
+ "serde_path_to_error",
+ "serde_urlencoded",
  "sync_wrapper",
  "tokio",
  "tower",
  "tower-layer",
  "tower-service",
+ "tracing",
 ]
 
 [[package]]
@@ -531,6 +535,7 @@ dependencies = [
  "sync_wrapper",
  "tower-layer",
  "tower-service",
+ "tracing",
 ]
 
 [[package]]
@@ -2874,6 +2879,17 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573"
 
+[[package]]
+name = "http-test-server"
+version = "1.0.0"
+dependencies = [
+ "anyhow",
+ "axum 0.7.4",
+ "futures",
+ "serde",
+ "tokio",
+]
+
 [[package]]
 name = "httparse"
 version = "1.8.0"
@@ -5462,6 +5478,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_path_to_error"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c"
+dependencies = [
+ "itoa 1.0.10",
+ "serde",
+]
+
 [[package]]
 name = "serde_repr"
 version = "0.1.18"

rust/Cargo.toml

@@ -15,6 +15,7 @@ members = [
   "relay",
   "gui-client/src-tauri",
   "http-health-check",
+  "http-test-server",
 ]
 
 resolver = "2"

rust/http-test-server/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "http-test-server"
+# mark:automatic-version
+version = "1.0.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1"
+axum = { version = "0.7.3", features = ["http1", "tokio"] }
+tokio = { version = "1.36.0", features = ["net"] }
+serde = {version = "1", features = ["derive"]}
+futures = "0.3"

rust/http-test-server/README.MD

@@ -0,0 +1,3 @@
+# HTTP test server
+
+An HTTP server that exposes endpoints for simulating file downloads.

rust/http-test-server/src/main.rs

@@ -0,0 +1,43 @@
+use anyhow::{Context, Result};
+use axum::{
+    body::{Body, Bytes},
+    extract::Query,
+    http::Response,
+    response::IntoResponse,
+    routing::get,
+    Router,
+};
+use futures::StreamExt;
+use std::{convert::Infallible, net::Ipv4Addr};
+use tokio::net::TcpListener;
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> Result<()> {
+    let port = std::env::var("PORT")
+        .context("Missing env var `PORT`")?
+        .parse::<u16>()?;
+
+    let router = Router::new().route("/bytes", get(byte_stream));
+    let listener = TcpListener::bind((Ipv4Addr::UNSPECIFIED, port)).await?;
+
+    axum::serve(listener, router).await?;
+
+    Ok(())
+}
+
+#[derive(serde::Deserialize)]
+struct Params {
+    num: usize,
+}
+
+async fn byte_stream(Query(params): Query<Params>) -> impl IntoResponse {
+    let body = Body::from_stream(
+        futures::stream::repeat(0)
+            .take(params.num)
+            .chunks(100)
+            .map(|slice| Bytes::copy_from_slice(&slice))
+            .map(Result::<_, Infallible>::Ok),
+    );
+
+    Response::new(body)
+}

scripts/tests/direct-download-roaming-network.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "./scripts/tests/lib.sh"
+
+# Download 10MB at a max rate of 1MB/s. Shouldn't take longer than 12 seconds (allows for 2s of restablishing)
+docker compose exec -it client sh -c \
+    "curl \
+        --fail \
+        --max-time 12 \
+        --limit-rate 1M http://download.httpbin/bytes?num=10000000" > download.file &
+
+DOWNLOAD_PID=$!
+
+sleep 3 # Download a bit
+
+docker network disconnect firezone_app firezone-client-1 # Disconnect the client
+sleep 1
+
+docker network connect firezone_app firezone-client-1 --ip 172.28.0.200 # Reconnect client with a different IP
+sudo kill -s HUP "$(ps -C firezone-linux-client -o pid=)" # Send SIGHUP, triggering `reconnect` internally
+
+wait $DOWNLOAD_PID || {
+    echo "Download process failed"
+    exit 1
+}
+
+known_checksum="f5e02aa71e67f41d79023a128ca35bad86cf7b6656967bfe0884b3a3c4325eaf"
+computed_checksum=$(sha256sum download.file | awk '{ print $1 }')
+
+if [[ "$computed_checksum" != "$known_checksum" ]]; then
+    echo "Checksum of downloaded file does not match"
+    exit 1
+fi