diff --git a/.github/workflows/_build_artifacts.yml b/.github/workflows/_build_artifacts.yml index 41330ccc0..56973b7b3 100644 --- a/.github/workflows/_build_artifacts.yml +++ b/.github/workflows/_build_artifacts.yml @@ -137,8 +137,7 @@ jobs: # Exclude debug builds for non-amd64 targets since they won't be used. - {stage: debug, arch: {platform: linux/arm/v7}} - {stage: debug, arch: {platform: linux/arm64}} - # Exclude snownet-tests and http-test-server from perf image builds - - {image_prefix: perf, name: {package: snownet-tests}} + # Exclude http-test-server from perf image builds - {image_prefix: perf, name: {package: http-test-server}} arch: @@ -169,9 +168,6 @@ jobs: release_name: gateway-1.2.0 # mark:next-gateway-version version: 1.2.0 - - package: snownet-tests - artifact: snownet-tests - image_name: snownet-tests - package: http-test-server artifact: http-test-server image_name: http-test-server @@ -343,9 +339,8 @@ jobs: image_prefix: - ${{ inputs.image_prefix }} - # Exclude snownet-tests and http-test-server from perf image builds + # Exclude http-test-server from perf image builds exclude: - - {image_prefix: perf, image: {name: snownet-tests}} - {image_prefix: perf, image: {name: http-test-server}} image: @@ -356,7 +351,6 @@ jobs: - name: client # mark:next-client-version version: 1.0.6 - - name: snownet-tests - name: http-test-server steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ebbc9a247..632718bd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,36 +102,6 @@ jobs: relay_image: ${{ needs.build-artifacts.outputs.relay_image }} http_test_server_image: ${{ needs.build-artifacts.outputs.http_test_server_image }} - snownet-tests: - needs: build-artifacts - if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }} - name: snownet-tests-${{ matrix.name }} - runs-on: ubuntu-22.04 - permissions: - contents: read - id-token: write - pull-requests: write - env: - RELAY_TAG: ${{ github.sha }} - SNOWNET_TAG: ${{ github.sha }} - strategy: - fail-fast: false - matrix: - name: - - lan - - wan-hp - - wan-relay - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/gcp-docker-login - id: login - with: - project: firezone-staging - - name: Run docker-compose.${{ matrix.name }}.yml test - run: | - sudo sysctl -w vm.overcommit_memory=1 - timeout 600 docker compose -f rust/tests/snownet-tests/docker-compose.${{ matrix.name }}.yml up --exit-code-from dialer --abort-on-container-exit - compatibility-tests: strategy: fail-fast: false diff --git a/rust/Cargo.lock b/rust/Cargo.lock index fa61504c8..1bf5954e7 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -171,15 +171,6 @@ dependencies = [ "x11rb", ] -[[package]] -name = "array-init" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23589ecb866b460d3a0f1278834750268c607e8e28a1b982c907219f3178cd72" -dependencies = [ - "nodrop", -] - [[package]] name = "ascii" version = "1.1.0" @@ -779,7 +770,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3431df59f28accaf4cb4eed4a9acc66bea3f3c3753aa6cdc2f024174ef232af7" dependencies = [ - "smallvec 1.13.2", + "smallvec", ] [[package]] @@ -788,7 +779,7 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa50868b64a9a6fda9d593ce778849ea8715cd2a3d2cc17ffdb4a2f2f2f1961d" dependencies = [ - "smallvec 1.13.2", + "smallvec", "target-lexicon", ] @@ -973,11 +964,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ "bytes", - "futures-core", "memchr", - "pin-project-lite", - "tokio", - "tokio-util", ] [[package]] @@ -1275,7 +1262,7 @@ dependencies = [ "phf 0.8.0", "proc-macro2", "quote", - "smallvec 1.13.2", + "smallvec", "syn 1.0.109", ] @@ -2001,7 +1988,7 @@ dependencies = [ "secrecy", "serde", "sha2", - "smallvec 1.13.2", + "smallvec", "socket-factory", "socket2", "stun_codec", @@ -2476,7 +2463,7 @@ dependencies = [ "gobject-sys", "libc", "once_cell", - "smallvec 1.13.2", + "smallvec", "thiserror", ] @@ -2835,7 +2822,7 @@ dependencies = [ "httpdate", "itoa 1.0.11", "pin-project-lite", - "smallvec 1.13.2", + "smallvec", "tokio", "want", ] @@ -3440,12 +3427,6 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md5" version = "0.7.0" @@ -4222,7 +4203,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "smallvec 1.13.2", + "smallvec", "windows-targets 0.48.5", ] @@ -4871,50 +4852,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9" -[[package]] -name = "redis" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d7a6955c7511f60f3ba9e86c6d02b3c3f144f8c24b288d1f4e18074ab8bbec" -dependencies = [ - "async-trait", - "bytes", - "combine", - "futures-util", - "itoa 1.0.11", - "percent-encoding", - "pin-project-lite", - "ryu", - "sha1_smol", - "socket2", - "tokio", - "tokio-util", - "url", -] - -[[package]] -name = "redis-macros" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8b5407866b6626d251b18c878f043d37f43124680f26a806595a61714ab049a" -dependencies = [ - "redis", - "redis-macros-derive", - "serde", - "serde_json", -] - -[[package]] -name = "redis-macros-derive" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dfe1dc77e38e260bbd53e98d3aec64add3cdf5d773e38d344c63660196117f5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -5317,7 +5254,7 @@ dependencies = [ "phf_codegen 0.8.0", "precomputed-hash", "servo_arc", - "smallvec 1.13.2", + "smallvec", "thin-slice", ] @@ -5339,17 +5276,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-hex" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca37e3e4d1b39afd7ff11ee4e947efae85adfddf4841787bfa47c470e96dc26d" -dependencies = [ - "array-init", - "serde", - "smallvec 0.6.14", -] - [[package]] name = "serde_assert" version = "0.7.1" @@ -5508,12 +5434,6 @@ dependencies = [ "cc", ] -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - [[package]] name = "sha2" version = "0.10.8" @@ -5564,15 +5484,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "smallvec" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - [[package]] name = "smallvec" version = "1.13.2" @@ -5612,30 +5523,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "snownet-tests" -version = "0.1.0" -dependencies = [ - "anyhow", - "boringtun", - "firezone-logging", - "futures", - "hex", - "ip-packet", - "pnet_packet", - "rand 0.8.5", - "redis", - "redis-macros", - "secrecy", - "serde", - "serde-hex", - "serde_json", - "snownet", - "system-info", - "tokio", - "tracing", -] - [[package]] name = "socket-factory" version = "0.1.0" @@ -5925,16 +5812,6 @@ dependencies = [ "version-compare 0.2.0", ] -[[package]] -name = "system-info" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6649e0c93f64c8ebcb719ba8e8e6fc581258350b67387f440161bfcd775a0ca" -dependencies = [ - "libc", - "windows-sys 0.36.1", -] - [[package]] name = "tao" version = "0.16.8" @@ -6399,7 +6276,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -6676,7 +6552,7 @@ dependencies = [ "once_cell", "opentelemetry", "opentelemetry_sdk", - "smallvec 1.13.2", + "smallvec", "tracing", "tracing-core", "tracing-log", @@ -6734,7 +6610,7 @@ dependencies = [ "serde", "serde_json", "sharded-slab", - "smallvec 1.13.2", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -7458,19 +7334,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -7601,12 +7464,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.37.0" @@ -7637,12 +7494,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.37.0" @@ -7679,12 +7530,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.37.0" @@ -7715,12 +7560,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.37.0" @@ -7769,12 +7608,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.37.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 91c44423b..77718b96e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -17,7 +17,6 @@ members = [ "socket-factory", "tests/gui-smoke-test", "tests/http-test-server", - "tests/snownet-tests", "tun" ] diff --git a/rust/Dockerfile b/rust/Dockerfile index 0818c2c56..c1daf36e0 100644 --- a/rust/Dockerfile +++ b/rust/Dockerfile @@ -80,10 +80,6 @@ COPY ./docker-init.sh ./docker-init.sh FROM runtime_base AS runtime_http-test-server COPY ./docker-init.sh ./docker-init.sh -# snownet-tests specific runtime base image -FROM runtime_base AS runtime_snownet-tests -COPY ./docker-init.sh ./docker-init.sh - # Funnel package specific base image back into `runtime` FROM runtime_${PACKAGE} AS runtime diff --git a/rust/tests/snownet-tests/Cargo.toml b/rust/tests/snownet-tests/Cargo.toml deleted file mode 100644 index f125154e3..000000000 --- a/rust/tests/snownet-tests/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "snownet-tests" -version = "0.1.0" -edition = "2021" - -[dependencies] -anyhow = "1" -boringtun = { workspace = true } -firezone-logging = { workspace = true } -futures = "0.3" -hex = "0.4" -ip-packet = { workspace = true } -pnet_packet = { version = "0.35" } -rand = "0.8" -redis = { version = "0.25.4", default-features = false, features = ["tokio-comp"] } -redis-macros = "0.3.0" -secrecy = { workspace = true } -serde = { version = "1", features = ["derive"] } -serde-hex = "0.1.0" -serde_json = "1" -snownet = { workspace = true } -system-info = { version = "0.1.2", features = ["std"] } -tokio = { workspace = true, features = ["full"] } -tracing = "0.1" - -[lints] -workspace = true diff --git a/rust/tests/snownet-tests/README.md b/rust/tests/snownet-tests/README.md deleted file mode 100644 index 16b1c5433..000000000 --- a/rust/tests/snownet-tests/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# snownet integration tests - -This directory contains Docker-based integration tests for the `snownet` crate. -Each integration test setup is a dedicated docker-compose file. - -## Running - -To run one of these tests, use the following command: - -```shell -sudo docker compose -f ./docker-compose.lan.yml up --exit-code-from dialer --abort-on-container-exit --build -``` - -This will force a re-build of the containers and exit with 0 if everything works correctly. - -## Design - -Each file consists of at least: - -- A dialer -- A listener -- A redis server - -Redis acts as the signalling channel. -Dialer and listener use it to exchange offers & answers as well as ICE candidates. - -The various files simulate different network environments. -We use nftables to simulate NATs and / or force the use of TURN servers. diff --git a/rust/tests/snownet-tests/docker-compose.lan.yml b/rust/tests/snownet-tests/docker-compose.lan.yml deleted file mode 100644 index d7d5c67a7..000000000 --- a/rust/tests/snownet-tests/docker-compose.lan.yml +++ /dev/null @@ -1,140 +0,0 @@ -# This test environment partitions has dialer and listener on the same subnet. -# The relay acts only as a STUN server and sits in a different network. -# This allows us to test that our automatic discovery of host candidates makes a local connection possible. - -version: "3.8" -name: lan-integration-test - -services: - dialer: - build: - target: debug - context: .. - args: - PACKAGE: snownet-tests - cache_from: - - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/snownet-tests:main - image: ${SNOWNET_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/debug/snownet-tests}:${SNOWNET_TAG:-main} - environment: - ROLE: "dialer" - cap_add: - - NET_ADMIN - entrypoint: /bin/sh - command: - - -c - - | - set -ex - - ROUTER_IP=$$(dig +short router) - INTERNET_SUBNET=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/networks/lan-integration-test_wan | jq -r '.IPAM.Config[0].Subnet') - - ip route add $$INTERNET_SUBNET via $$ROUTER_IP dev eth0 - - export STUN_SERVER=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/lan-integration-test-relay-1/json | jq -r '.NetworkSettings.Networks."lan-integration-test_wan".IPAddress') - export REDIS_HOST=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/lan-integration-test-redis-1/json | jq -r '.NetworkSettings.Networks."lan-integration-test_wan".IPAddress') - - snownet-tests - depends_on: - - router - - redis - networks: - - lan - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - router: - init: true - build: - context: ./router - cap_add: - - NET_ADMIN - networks: - - lan - - wan - - listener: - build: - target: debug - context: .. - args: - PACKAGE: snownet-tests - cache_from: - - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/snownet-tests:main - image: ${SNOWNET_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/debug/snownet-tests}:${SNOWNET_TAG:-main} - init: true - environment: - ROLE: "listener" - entrypoint: /bin/sh - command: - - -c - - | - set -ex - - ROUTER_IP=$$(dig +short router) - INTERNET_SUBNET=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/networks/lan-integration-test_wan | jq -r '.IPAM.Config[0].Subnet') - - ip route add $$INTERNET_SUBNET via $$ROUTER_IP dev eth0 - - export STUN_SERVER=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/lan-integration-test-relay-1/json | jq -r '.NetworkSettings.Networks."lan-integration-test_wan".IPAddress') - export REDIS_HOST=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/lan-integration-test-redis-1/json | jq -r '.NetworkSettings.Networks."lan-integration-test_wan".IPAddress') - - snownet-tests - cap_add: - - NET_ADMIN - depends_on: - - router - - redis - networks: - - lan - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - relay: - environment: - LOWEST_PORT: 55555 - HIGHEST_PORT: 55666 - RUST_LOG: "debug" - RUST_BACKTRACE: 1 - build: - target: debug - context: .. - cache_from: - - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/relay:main - args: - PACKAGE: firezone-relay - image: ${RELAY_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/debug/relay}:${RELAY_TAG:-main} - init: true - healthcheck: - test: ["CMD-SHELL", "lsof -i UDP | grep firezone-relay"] - start_period: 20s - interval: 30s - retries: 5 - timeout: 5s - entrypoint: /bin/sh - command: - - -c - - | - set -ex; - export PUBLIC_IP4_ADDR=$(ip -json addr show eth0 | jq '.[0].addr_info[0].local' -r) - - firezone-relay - ports: - # NOTE: Only 111 ports are used for local dev / testing because Docker Desktop - # allocates a userland proxy process for each forwarded port X_X. - # - # Large ranges here will bring your machine to its knees. - - "55555-55666:55555-55666/udp" - - 3478:3478/udp - networks: - - wan - - redis: - image: "redis:7-alpine" - healthcheck: - test: ["CMD-SHELL", "echo 'ready';"] - networks: - - wan - -networks: - lan: - wan: diff --git a/rust/tests/snownet-tests/docker-compose.wan-hp.yml b/rust/tests/snownet-tests/docker-compose.wan-hp.yml deleted file mode 100644 index b47786257..000000000 --- a/rust/tests/snownet-tests/docker-compose.wan-hp.yml +++ /dev/null @@ -1,150 +0,0 @@ -# This test environment partitions dialer and listener into different subnets. -# The routers use a persistent port mapping, allowing the two clients to hole-punch a direct connection. - -version: "3.8" -name: wan-hp-integration-test - -services: - dialer: - build: - target: debug - context: .. - args: - PACKAGE: snownet-tests - cache_from: - - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/snownet-tests:main - image: ${SNOWNET_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/debug/snownet-tests}:${SNOWNET_TAG:-main} - environment: - ROLE: "dialer" - cap_add: - - NET_ADMIN - entrypoint: /bin/sh - command: - - -c - - | - set -ex - - ROUTER_IP=$$(dig +short dialer_router) - INTERNET_SUBNET=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/networks/wan-hp-integration-test_wan | jq -r '.IPAM.Config[0].Subnet') - - ip route add $$INTERNET_SUBNET via $$ROUTER_IP dev eth0 - - export STUN_SERVER=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/wan-hp-integration-test-relay-1/json | jq -r '.NetworkSettings.Networks."wan-hp-integration-test_wan".IPAddress') - export REDIS_HOST=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/wan-hp-integration-test-redis-1/json | jq -r '.NetworkSettings.Networks."wan-hp-integration-test_wan".IPAddress') - - snownet-tests - depends_on: - - dialer_router - - redis - networks: - - lan1 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - dialer_router: - init: true - build: - context: ./router - cap_add: - - NET_ADMIN - networks: - - lan1 - - wan - - listener: - build: - target: debug - context: .. - args: - PACKAGE: snownet-tests - cache_from: - - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/snownet-tests:main - image: ${SNOWNET_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/debug/snownet-tests}:${SNOWNET_TAG:-main} - init: true - environment: - ROLE: "listener" - entrypoint: /bin/sh - command: - - -c - - | - set -ex - - ROUTER_IP=$$(dig +short listener_router) - INTERNET_SUBNET=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/networks/wan-hp-integration-test_wan | jq -r '.IPAM.Config[0].Subnet') - - ip route add $$INTERNET_SUBNET via $$ROUTER_IP dev eth0 - - export STUN_SERVER=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/wan-hp-integration-test-relay-1/json | jq -r '.NetworkSettings.Networks."wan-hp-integration-test_wan".IPAddress') - export REDIS_HOST=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/wan-hp-integration-test-redis-1/json | jq -r '.NetworkSettings.Networks."wan-hp-integration-test_wan".IPAddress') - - snownet-tests - cap_add: - - NET_ADMIN - depends_on: - - listener_router - - redis - networks: - - lan2 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - listener_router: - init: true - build: - context: ./router - cap_add: - - NET_ADMIN - networks: - - lan2 - - wan - - relay: - environment: - LOWEST_PORT: 55555 - HIGHEST_PORT: 55666 - RUST_LOG: "debug" - RUST_BACKTRACE: 1 - build: - target: debug - context: .. - cache_from: - - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/relay:main - args: - PACKAGE: firezone-relay - image: ${RELAY_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/debug/relay}:${RELAY_TAG:-main} - init: true - healthcheck: - test: ["CMD-SHELL", "lsof -i UDP | grep firezone-relay"] - start_period: 20s - interval: 30s - retries: 5 - timeout: 5s - entrypoint: /bin/sh - command: - - -c - - | - set -ex; - export PUBLIC_IP4_ADDR=$(ip -json addr show eth0 | jq '.[0].addr_info[0].local' -r) - - firezone-relay - ports: - # NOTE: Only 111 ports are used for local dev / testing because Docker Desktop - # allocates a userland proxy process for each forwarded port X_X. - # - # Large ranges here will bring your machine to its knees. - - "55555-55666:55555-55666/udp" - - 3478:3478/udp - networks: - - wan - - redis: - image: "redis:7-alpine" - healthcheck: - test: ["CMD-SHELL", "echo 'ready';"] - networks: - - wan - -networks: - lan1: - lan2: - wan: diff --git a/rust/tests/snownet-tests/docker-compose.wan-relay.yml b/rust/tests/snownet-tests/docker-compose.wan-relay.yml deleted file mode 100644 index a6cc8340b..000000000 --- a/rust/tests/snownet-tests/docker-compose.wan-relay.yml +++ /dev/null @@ -1,154 +0,0 @@ -# This test environment partitions dialer and listener into different subnets. -# Their router is configured to use `fully-random` port mapping, making hole-punching impossible. - -version: "3.8" -name: wan-relay-integration-test - -services: - dialer: - build: - target: debug - context: .. - args: - PACKAGE: snownet-tests - cache_from: - - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/snownet-tests:main - image: ${SNOWNET_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/debug/snownet-tests}:${SNOWNET_TAG:-main} - environment: - ROLE: "dialer" - cap_add: - - NET_ADMIN - entrypoint: /bin/sh - command: - - -c - - | - set -ex - - ROUTER_IP=$$(dig +short dialer_router) - INTERNET_SUBNET=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/networks/wan-relay-integration-test_wan | jq -r '.IPAM.Config[0].Subnet') - - ip route add $$INTERNET_SUBNET via $$ROUTER_IP dev eth0 - - export TURN_SERVER=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/wan-relay-integration-test-relay-1/json | jq -r '.NetworkSettings.Networks."wan-relay-integration-test_wan".IPAddress') - export REDIS_HOST=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/wan-relay-integration-test-redis-1/json | jq -r '.NetworkSettings.Networks."wan-relay-integration-test_wan".IPAddress') - - snownet-tests - depends_on: - - dialer_router - - redis - networks: - - lan1 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - dialer_router: - init: true - build: - context: ./router - environment: - NAT_BEHAVIOUR: fully-random - cap_add: - - NET_ADMIN - networks: - - lan1 - - wan - - listener: - build: - target: debug - context: .. - args: - PACKAGE: snownet-tests - cache_from: - - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/snownet-tests:main - image: ${SNOWNET_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/debug/snownet-tests}:${SNOWNET_TAG:-main} - init: true - environment: - ROLE: "listener" - entrypoint: /bin/sh - command: - - -c - - | - set -ex - - ROUTER_IP=$$(dig +short listener_router) - INTERNET_SUBNET=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/networks/wan-relay-integration-test_wan | jq -r '.IPAM.Config[0].Subnet') - - ip route add $$INTERNET_SUBNET via $$ROUTER_IP dev eth0 - - export TURN_SERVER=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/wan-relay-integration-test-relay-1/json | jq -r '.NetworkSettings.Networks."wan-relay-integration-test_wan".IPAddress') - export REDIS_HOST=$$(curl --fail --silent --unix-socket /var/run/docker.sock http://localhost/containers/wan-relay-integration-test-redis-1/json | jq -r '.NetworkSettings.Networks."wan-relay-integration-test_wan".IPAddress') - - snownet-tests - cap_add: - - NET_ADMIN - depends_on: - - listener_router - - redis - networks: - - lan2 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - listener_router: - init: true - build: - context: ./router - environment: - NAT_BEHAVIOUR: fully-random - cap_add: - - NET_ADMIN - networks: - - lan2 - - wan - - relay: - environment: - LOWEST_PORT: 55555 - HIGHEST_PORT: 55666 - RUST_LOG: "debug" - RUST_BACKTRACE: 1 - RNG_SEED: 0 - build: - target: debug - context: .. - cache_from: - - type=registry,ref=us-east1-docker.pkg.dev/firezone-staging/cache/relay:main - args: - PACKAGE: firezone-relay - image: ${RELAY_IMAGE:-us-east1-docker.pkg.dev/firezone-staging/firezone/debug/relay}:${RELAY_TAG:-main} - healthcheck: - test: ["CMD-SHELL", "lsof -i UDP | grep firezone-relay"] - start_period: 20s - interval: 30s - retries: 5 - timeout: 5s - entrypoint: /bin/sh - command: - - -c - - | - set -ex; - export PUBLIC_IP4_ADDR=$(ip -json addr show eth0 | jq '.[0].addr_info[0].local' -r) - - firezone-relay - ports: - # NOTE: Only 111 ports are used for local dev / testing because Docker Desktop - # allocates a userland proxy process for each forwarded port X_X. - # - # Large ranges here will bring your machine to its knees. - - "55555-55666:55555-55666/udp" - - 3478:3478/udp - networks: - - wan - - redis: - image: "redis:7-alpine" - healthcheck: - test: ["CMD-SHELL", "echo 'ready';"] - networks: - - wan - -networks: - lan1: - lan2: - wan: diff --git a/rust/tests/snownet-tests/router/Dockerfile b/rust/tests/snownet-tests/router/Dockerfile deleted file mode 100644 index 92e4570d5..000000000 --- a/rust/tests/snownet-tests/router/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM debian:12-slim - -ARG DEBIAN_FRONTEND=noninteractive -RUN --mount=type=cache,target=/var/cache/apt apt-get update && apt-get -y install iproute2 nftables conntrack - -COPY *.sh /scripts/ -RUN chmod +x /scripts/*.sh - -HEALTHCHECK CMD [ "sh", "-c", "test $(cat /tmp/setup_done) = 1" ] - -ENTRYPOINT ["./scripts/run.sh"] diff --git a/rust/tests/snownet-tests/router/README.md b/rust/tests/snownet-tests/router/README.md deleted file mode 100644 index 38ec6b6ff..000000000 --- a/rust/tests/snownet-tests/router/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Router - -This directory contains a Debian-based router implemented on top of nftables. - -It expects to be run with two network interfaces: - -- `eth1`: The "external" interface. -- `eth0`: The "internal" interface. - -The order of these interfaces depends on lexical sorting the docker networks names. - -The order of these is important. -The router cannot possibly know which one is which and thus assumes that `eth0` is the external one and `eth1` the internal one. -The firewall is set up to take incoming traffic on `eth1` and forward + masquerade it to `eth0`. - -It also expects an env variable `DELAY_MS` to be set and will apply this delay as part of the routing process[^1]. - -[^1]: This is done via `tc qdisc` which only works for egress traffic. To ensure the delay applies in both directions, we divide it by 2 and apply it on both interfaces. diff --git a/rust/tests/snownet-tests/router/run.sh b/rust/tests/snownet-tests/router/run.sh deleted file mode 100644 index 24c099597..000000000 --- a/rust/tests/snownet-tests/router/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -set -ex - -# Set up NAT -nft add table ip nat -nft add chain ip nat postrouting '{' type nat hook postrouting priority 100 \; '}' -nft add rule ip nat postrouting masquerade $NAT_BEHAVIOUR - -# Assumption after a long debugging session involving Gabi, Jamil and Thomas: -# On the same machine, the kernel cannot differentiate between incoming and outgoing packets across different network namespaces within the firewall and NAT mapping table. -# As a result, even UDP hole-punching is time-sensitive and we thus need to make sure that we first send a packet _out_ through the router before the other one is incoming. -# To achieve this, we set an absurdly high latency of 300ms for the WAN network. -tc qdisc add dev eth1 root netem delay 300ms - -echo "1" >/tmp/setup_done # This will be checked by our docker HEALTHCHECK - -conntrack --event --proto UDP --output timestamp # Display a real-time log of NAT events in the kernel. diff --git a/rust/tests/snownet-tests/src/main.rs b/rust/tests/snownet-tests/src/main.rs deleted file mode 100644 index bbb2526e2..000000000 --- a/rust/tests/snownet-tests/src/main.rs +++ /dev/null @@ -1,474 +0,0 @@ -use std::{ - collections::BTreeSet, - future::poll_fn, - net::{Ipv4Addr, SocketAddrV4}, - str::FromStr, - task::{Context, Poll}, - time::Instant, -}; - -use anyhow::{bail, Context as _, Result}; -use boringtun::x25519::StaticSecret; -use futures::{channel::mpsc, future::BoxFuture, FutureExt, SinkExt, StreamExt}; -use ip_packet::IpPacket; -use pnet_packet::{ip::IpNextHeaderProtocols, ipv4::Ipv4Packet}; -use redis::{aio::MultiplexedConnection, AsyncCommands}; -use secrecy::{ExposeSecret as _, Secret}; -use snownet::{Answer, ClientNode, Credentials, Node, Offer, RelaySocket, ServerNode}; -use tokio::{io::ReadBuf, net::UdpSocket}; - -const MAX_UDP_SIZE: usize = (1 << 16) - 1; - -#[tokio::main] -async fn main() -> Result<()> { - let _guard = - firezone_logging::test("info,boringtun=debug,str0m=debug,boringtun=debug,snownet=debug"); - - let role = std::env::var("ROLE") - .context("Missing ROLE env variable")? - .parse::()?; - - let listen_addr = system_info::NetworkInterfaces::new() - .context("Failed to get network interfaces")? - .iter() - .find_map(|i| i.addresses().find(|a| !a.ip.is_loopback())) - .context("Failed to find interface with non-loopback address")? - .ip - .to_std(); - - let relay_stun_only = std::env::var("STUN_SERVER") - .ok() - .map(|a| a.parse::()) - .transpose() - .context("Failed to parse `STUN_SERVER`")? - .map(|ip| { - ( - 1, - RelaySocket::V4(SocketAddrV4::new(ip, 3478)), - String::new(), - String::new(), - String::new(), - ) - }); - let relay_valid_turn = std::env::var("TURN_SERVER") - .ok() - .map(|a| a.parse::()) - .transpose() - .context("Failed to parse `TURN_SERVER`")? - .map(|ip| { - ( - 2, - RelaySocket::V4(SocketAddrV4::new(ip, 3478)), - "2000000000:client".to_owned(), // TODO: Use different credentials per role. - "+Qou8TSjw9q3JMnWET7MbFsQh/agwz/LURhpfX7a0hE".to_owned(), - "firezone".to_owned(), - ) - }); - let relays = BTreeSet::from_iter(relay_stun_only.into_iter().chain(relay_valid_turn)); - - tracing::info!(%listen_addr); - - let redis_host = std::env::var("REDIS_HOST").context("Missing REDIS_HOST env var")?; - - let redis_client = redis::Client::open(format!("redis://{redis_host}:6379"))?; - let mut redis_connection = redis_client.get_multiplexed_async_connection().await?; - - let socket = UdpSocket::bind((listen_addr, 0)).await?; - let private_key = StaticSecret::random_from_rng(rand::thread_rng()); - - // The source and dst of our dummy IP packet that we send via the wireguard tunnel. - let source = Ipv4Addr::new(172, 16, 0, 1); - let dst = Ipv4Addr::new(10, 0, 0, 1); - - match role { - Role::Dialer => { - let mut node = ClientNode::::new(private_key, rand::random()); - node.update_relays(BTreeSet::new(), &relays, Instant::now()); - - let offer = node.new_connection(1, Instant::now(), Instant::now()); - - redis_connection - .rpush( - "offers", - wire::Offer { - session_key: *offer.session_key.expose_secret(), - username: offer.credentials.username, - password: offer.credentials.password, - public_key: node.public_key().to_bytes(), - }, - ) - .await - .context("Failed to push offer")?; - - let answer = redis_connection - .blpop::<_, (String, wire::Answer)>("answers", 10.0) - .await - .context("Failed to pop answer")? - .1; - - node.accept_answer( - 1, - answer.public_key.into(), - Answer { - credentials: Credentials { - username: answer.username, - password: answer.password, - }, - }, - Instant::now(), - ); - - let rx = spawn_candidate_task(redis_connection.clone(), "listener_candidates"); - - let mut eventloop = Eventloop::new(socket, node, rx); - - let ping_body = rand::random::<[u8; 32]>(); - let mut start = Instant::now(); - - loop { - match poll_fn(|cx| eventloop.poll(cx)).await? { - Event::Incoming { conn, packet } => { - anyhow::ensure!(conn == 1); - anyhow::ensure!( - packet - == IpPacket::Ipv4(ip4_udp_ping_packet( - dst, - source, - packet.udp_payload() - )) - ); // Expect the listener to flip src and dst - - let rtt = start.elapsed(); - - tracing::info!("RTT is {rtt:?}"); - - return Ok(()); - } - Event::SignalIceCandidate { conn, candidate } => { - redis_connection - .rpush("dialer_candidates", wire::Candidate { conn, candidate }) - .await - .context("Failed to push candidate")?; - } - Event::ConnectionEstablished { conn } => { - start = Instant::now(); - eventloop - .send_to(conn, ip4_udp_ping_packet(source, dst, &ping_body).into())?; - } - Event::ConnectionFailed { conn } => { - anyhow::bail!("Failed to establish connection: {conn}"); - } - } - } - } - Role::Listener => { - let mut node = ServerNode::::new(private_key, rand::random()); - node.update_relays(BTreeSet::new(), &relays, Instant::now()); - - let offer = redis_connection - .blpop::<_, (String, wire::Offer)>("offers", 10.0) - .await - .context("Failed to pop offer")? - .1; - - let answer = node.accept_connection( - 1, - Offer { - session_key: Secret::new(offer.session_key), - credentials: Credentials { - username: offer.username, - password: offer.password, - }, - }, - offer.public_key.into(), - Instant::now(), - ); - - redis_connection - .rpush( - "answers", - wire::Answer { - public_key: node.public_key().to_bytes(), - username: answer.credentials.username, - password: answer.credentials.password, - }, - ) - .await - .context("Failed to push answer")?; - - let rx = spawn_candidate_task(redis_connection.clone(), "dialer_candidates"); - - let mut eventloop = Eventloop::new(socket, node, rx); - - loop { - match poll_fn(|cx| eventloop.poll(cx)).await? { - Event::Incoming { conn, packet } => { - eventloop.send_to( - conn, - ip4_udp_ping_packet(dst, source, packet.udp_payload()).into(), - )?; - } - Event::SignalIceCandidate { conn, candidate } => { - redis_connection - .rpush("listener_candidates", wire::Candidate { conn, candidate }) - .await - .context("Failed to push candidate")?; - } - Event::ConnectionEstablished { .. } => {} - Event::ConnectionFailed { conn } => { - anyhow::bail!("Failed to establish connection: {conn}"); - } - } - } - } - }; -} - -fn spawn_candidate_task( - mut conn: MultiplexedConnection, - topic: &'static str, -) -> mpsc::Receiver { - let (mut sender, receiver) = mpsc::channel(0); - tokio::spawn(async move { - loop { - let candidate = conn - .blpop::<_, Option<(String, wire::Candidate)>>(topic, 1.0) - .await - .unwrap(); - - if let Some((_, candidate)) = candidate { - sender.send(candidate).await.unwrap(); - } - } - }); - - receiver -} - -fn ip4_udp_ping_packet(source: Ipv4Addr, dst: Ipv4Addr, body: &[u8]) -> Ipv4Packet<'static> { - assert_eq!(body.len(), 32); - - let mut packet_buffer = [0u8; 60]; - - let mut ip4_header = - pnet_packet::ipv4::MutableIpv4Packet::new(&mut packet_buffer[..20]).unwrap(); - ip4_header.set_version(4); - ip4_header.set_source(source); - ip4_header.set_destination(dst); - ip4_header.set_next_level_protocol(IpNextHeaderProtocols::Udp); - ip4_header.set_ttl(10); - ip4_header.set_total_length(20 + 8 + 32); // IP4 + UDP + payload. - ip4_header.set_header_length(5); // Length is in number of 32bit words, i.e. 5 means 20 bytes. - ip4_header.set_checksum(pnet_packet::ipv4::checksum(&ip4_header.to_immutable())); - - let mut udp_header = - pnet_packet::udp::MutableUdpPacket::new(&mut packet_buffer[20..28]).unwrap(); - udp_header.set_source(9999); - udp_header.set_destination(9999); - udp_header.set_length(8 + 32); - udp_header.set_checksum(0); // Not necessary for IPv4, let's keep it simple. - - packet_buffer[28..60].copy_from_slice(body); - - Ipv4Packet::owned(packet_buffer.to_vec()).unwrap() -} - -mod wire { - #[derive( - serde::Serialize, - serde::Deserialize, - redis_macros::FromRedisValue, - redis_macros::ToRedisArgs, - )] - pub struct Offer { - #[serde(with = "serde_hex::SerHex::")] - pub session_key: [u8; 32], - #[serde(with = "serde_hex::SerHex::")] - pub public_key: [u8; 32], - pub username: String, - pub password: String, - } - - #[derive( - serde::Serialize, - serde::Deserialize, - redis_macros::FromRedisValue, - redis_macros::ToRedisArgs, - )] - pub struct Answer { - #[serde(with = "serde_hex::SerHex::")] - pub public_key: [u8; 32], - pub username: String, - pub password: String, - } - - #[derive( - serde::Serialize, - serde::Deserialize, - redis_macros::FromRedisValue, - redis_macros::ToRedisArgs, - Debug, - )] - pub struct Candidate { - pub conn: u64, - pub candidate: String, - } -} - -enum Role { - Dialer, - Listener, -} - -impl FromStr for Role { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s { - "dialer" => Ok(Self::Dialer), - "listener" => Ok(Self::Listener), - other => bail!("unknown role: {other}"), - } - } -} - -struct Eventloop { - socket: UdpSocket, - pool: Node, - timeout: BoxFuture<'static, Instant>, - candidate_rx: mpsc::Receiver, - read_buffer: Box<[u8; MAX_UDP_SIZE]>, - write_buffer: Box<[u8; MAX_UDP_SIZE]>, -} - -impl Eventloop { - fn new( - socket: UdpSocket, - pool: Node, - candidate_rx: mpsc::Receiver, - ) -> Self { - Self { - socket, - pool, - timeout: sleep_until(Instant::now()).boxed(), - read_buffer: Box::new([0u8; MAX_UDP_SIZE]), - write_buffer: Box::new([0u8; MAX_UDP_SIZE]), - candidate_rx, - } - } - - fn send_to(&mut self, id: u64, packet: IpPacket<'_>) -> Result<()> { - let Some(transmit) = self.pool.encapsulate(id, packet, Instant::now())? else { - return Ok(()); - }; - - tracing::trace!(target = "wire::out", to = %transmit.dst, packet = %hex::encode(&transmit.payload)); - - self.socket.try_send_to(&transmit.payload, transmit.dst)?; - - Ok(()) - } - - fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { - while let Some(transmit) = self.pool.poll_transmit() { - tracing::trace!(target = "wire::out", to = %transmit.dst, packet = %hex::encode(&transmit.payload)); - - if let Some(src) = transmit.src { - assert_eq!(src, self.socket.local_addr()?); - } - - self.socket.try_send_to(&transmit.payload, transmit.dst)?; - } - - match self.pool.poll_event() { - Some(snownet::Event::NewIceCandidate { - connection, - candidate, - }) => { - return Poll::Ready(Ok(Event::SignalIceCandidate { - conn: connection, - candidate, - })) - } - Some(snownet::Event::ConnectionEstablished(conn)) => { - return Poll::Ready(Ok(Event::ConnectionEstablished { conn })) - } - Some(snownet::Event::ConnectionFailed(conn)) => { - return Poll::Ready(Ok(Event::ConnectionFailed { conn })) - } - Some( - snownet::Event::InvalidateIceCandidate { .. } - | snownet::Event::ConnectionClosed { .. }, - ) - | None => {} - } - - if let Poll::Ready(Some(wire::Candidate { conn, candidate })) = - self.candidate_rx.poll_next_unpin(cx) - { - self.pool - .add_remote_candidate(conn, candidate, Instant::now()); - - cx.waker().wake_by_ref(); - return Poll::Pending; - } - - if let Poll::Ready(instant) = self.timeout.poll_unpin(cx) { - self.pool.handle_timeout(instant); - if let Some(timeout) = self.pool.poll_timeout() { - self.timeout = sleep_until(timeout).boxed(); - } - - cx.waker().wake_by_ref(); - return Poll::Pending; - } - - let mut read_buf = ReadBuf::new(self.read_buffer.as_mut()); - if let Poll::Ready(from) = self.socket.poll_recv_from(cx, &mut read_buf)? { - let packet = read_buf.filled(); - - tracing::trace!(target = "wire::in", %from, packet = %hex::encode(packet)); - - if let Some((conn, packet)) = self.pool.decapsulate( - self.socket.local_addr()?, - from, - packet, - Instant::now(), - self.write_buffer.as_mut(), - )? { - return Poll::Ready(Ok(Event::Incoming { - conn, - packet: packet.to_immutable().to_owned(), - })); - } - - cx.waker().wake_by_ref(); - return Poll::Pending; - } - - Poll::Pending - } -} - -enum Event { - Incoming { - conn: u64, - packet: IpPacket<'static>, - }, - SignalIceCandidate { - conn: u64, - candidate: String, - }, - ConnectionEstablished { - conn: u64, - }, - ConnectionFailed { - conn: u64, - }, -} - -async fn sleep_until(deadline: Instant) -> Instant { - tokio::time::sleep_until(deadline.into()).await; - - deadline -}