From e47c1766bf2cdbbb432b328e30c3ab090ef466de Mon Sep 17 00:00:00 2001 From: Thomas Eizinger Date: Thu, 15 Feb 2024 00:55:28 +1100 Subject: [PATCH] ci: move tests to bash scripts (#3648) This improves maintenance because we can now use a regular matrix for the integration tests and one can locally use tools like shellcheck or a `bash-lsp` during development. --------- Signed-off-by: Jamil Co-authored-by: Jamil --- .github/workflows/ci.yml | 156 ++++++------------ scripts/tests/direct-ping-portal-down.sh | 13 ++ .../tests/direct-ping-portal-relay-down.sh | 13 ++ scripts/tests/direct-ping-portal-restart.sh | 13 ++ .../{dns-nm-setup.bash => tests/dns-nm.sh} | 0 scripts/tests/lib.sh | 15 ++ scripts/tests/relayed-ping-portal-down.sh | 16 ++ scripts/tests/relayed-ping-portal-restart.sh | 16 ++ 8 files changed, 137 insertions(+), 105 deletions(-) create mode 100755 scripts/tests/direct-ping-portal-down.sh create mode 100755 scripts/tests/direct-ping-portal-relay-down.sh create mode 100755 scripts/tests/direct-ping-portal-restart.sh rename scripts/{dns-nm-setup.bash => tests/dns-nm.sh} (100%) create mode 100755 scripts/tests/lib.sh create mode 100755 scripts/tests/relayed-ping-portal-down.sh create mode 100755 scripts/tests/relayed-ping-portal-restart.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9a6f173f..179cccc57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -172,7 +172,54 @@ jobs: timeout 600 docker compose -f rust/snownet-tests/${{ matrix.file }} up --exit-code-from dialer --abort-on-container-exit integration-tests: - name: integration-tests-${{ matrix.test_name }} + name: integration-tests-${{ matrix.test }} + needs: build-images + runs-on: ubuntu-22.04 + permissions: + contents: read + id-token: write + pull-requests: write + env: + VERSION: ${{ github.sha }} + strategy: + fail-fast: false + matrix: + test: [ + direct-ping-portal-restart, + relayed-ping-portal-restart, + direct-ping-portal-down, + relayed-ping-portal-down, + direct-ping-portal-relay-down, + dns-nm, + ] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/gcp-docker-login + id: login + with: + project: firezone-staging + - name: Seed database + run: docker compose run elixir /bin/sh -c 'cd apps/domain && mix ecto.seed' + - name: Start docker compose in the background + run: docker compose up -d api web client relay gateway + + - run: ./scripts/tests/${{ matrix.test }}.sh + + - name: Show Client logs + if: "!cancelled()" + run: docker compose logs client + - name: Show Relay logs + if: "!cancelled()" + run: docker compose logs relay + - name: Show Gateway logs + if: "!cancelled()" + run: docker compose logs gateway + - name: Show API logs + if: "!cancelled()" + run: docker compose logs api + + performance-tests: + name: performance-tests-${{ matrix.test_name }} needs: build-images runs-on: ubuntu-22.04 permissions: @@ -185,111 +232,13 @@ jobs: fail-fast: false matrix: include: - # Define new integration tests here. These are executed in parallel in our CI. - # If you set `perf_test` to true, the throughput test will - # be activated and performance results will be uploaded as artifacts. - # If you leave `perf_test` blank or set it to false, the performance test will - # be skipped. - - # Basic connectivity tests - - test_name: direct-ping-portal-restart - setup: echo 'Noop' - execute: | - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 10 172.20.0.100 &>/dev/null; do true; done' - - # Restart portal - docker compose restart api - - # Wait for client to reconnect - sleep 5 - - # Ping again - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - test_name: relayed-ping-portal-restart - setup: | - # Disallow traffic between gateway and client container - sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP - sudo iptables -I FORWARD 1 -s 172.28.0.105 -d 172.28.0.100 -j DROP - execute: | - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - # Restart portal - docker compose restart api - - # Wait for client to reconnect - sleep 5 - - # Ping again - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - test_name: direct-ping-portal-down - setup: echo 'Noop' - execute: | - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 10 172.20.0.100 &>/dev/null; do true; done' - - # Stop portal - docker compose stop api - - # Wait for client to disconnect - sleep 5 - - # Ping again - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - test_name: relayed-ping-portal-down - setup: | - # Disallow traffic between gateway and client container - sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP - sudo iptables -I FORWARD 1 -s 172.28.0.105 -d 172.28.0.100 -j DROP - execute: | - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - # Stop portal - docker compose stop api - - # Wait for client to disconnect - sleep 5 - - # Ping again - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - test_name: direct-ping-portal-relay-down - setup: echo 'Noop' - execute: | - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 10 172.20.0.100 &>/dev/null; do true; done' - - # Stop api, relay - docker compose stop api relay - - # Wait for client to disconnect - sleep 5 - - # Ping again - docker compose exec -it client timeout 60 \ - sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - - test_name: dns-nm - setup: scripts/dns-nm-setup.bash - execute: echo 'Noop' - - # Performance Tests - test_name: direct-perf - # Set this to enable performance testing for this test - perf_test: true setup: echo 'Noop' execute: | # Establish a channel docker compose exec -it client timeout 60 \ sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - test_name: relayed-perf - # Set this to enable performance testing for this test - perf_test: true setup: | # Disallow traffic between gateway and client container sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP @@ -299,7 +248,6 @@ jobs: docker compose exec -it client timeout 60 \ sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - steps: - uses: actions/checkout@v4 - uses: ./.github/actions/gcp-docker-login @@ -323,7 +271,6 @@ jobs: - name: 'Execute test: ${{ matrix.test_name }}' run: ${{ matrix.execute }} - name: 'Performance test: ${{ matrix.test_name }}' - if: matrix.perf_test id: perfomance-test timeout-minutes: 5 run: | @@ -364,14 +311,13 @@ jobs: cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT" cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT" - name: 'Save performance test results: ${{ matrix.test_name }}' - if: matrix.perf_test uses: actions/upload-artifact@v4 with: name: '${{ matrix.test_name }}-iperf3results' path: /tmp/iperf3results - name: 'Download main branch performance test results: ${{ matrix.test_name }}' id: download-artifact - if: ${{ github.event_name == 'pull_request' && matrix.perf_test }} + if: ${{ github.event_name == 'pull_request' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -406,7 +352,7 @@ jobs: rm "${DESTINATION}.zip" - name: "Generate main branch metrics" id: main-perfomance-test - if: ${{ github.event_name == 'pull_request' && matrix.perf_test }} + if: ${{ github.event_name == 'pull_request' }} run: | cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT" cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT" @@ -426,7 +372,7 @@ jobs: - name: Update PR uses: actions/github-script@v7 id: perf-comment - if: ${{ github.event_name == 'pull_request' && matrix.perf_test }} + if: ${{ github.event_name == 'pull_request' }} with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/scripts/tests/direct-ping-portal-down.sh b/scripts/tests/direct-ping-portal-down.sh new file mode 100755 index 000000000..c467e6f9d --- /dev/null +++ b/scripts/tests/direct-ping-portal-down.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +source "./scripts/tests/lib.sh" + +client_ping_gateway + +docker compose stop api relay # Stop portal & relay + +sleep 5 # Wait for client to disconnect + +client_ping_gateway diff --git a/scripts/tests/direct-ping-portal-relay-down.sh b/scripts/tests/direct-ping-portal-relay-down.sh new file mode 100755 index 000000000..59e22e207 --- /dev/null +++ b/scripts/tests/direct-ping-portal-relay-down.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +source "./scripts/tests/lib.sh" + +client_ping_gateway + +docker compose stop api # Stop portal + +sleep 5 # Wait for client to disconnect + +client_ping_gateway diff --git a/scripts/tests/direct-ping-portal-restart.sh b/scripts/tests/direct-ping-portal-restart.sh new file mode 100755 index 000000000..52ef40d4c --- /dev/null +++ b/scripts/tests/direct-ping-portal-restart.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +source "./scripts/tests/lib.sh" + +client_ping_gateway + +docker compose restart api # Restart portal + +sleep 5 # Wait for client to reconnect + +client_ping_gateway diff --git a/scripts/dns-nm-setup.bash b/scripts/tests/dns-nm.sh similarity index 100% rename from scripts/dns-nm-setup.bash rename to scripts/tests/dns-nm.sh diff --git a/scripts/tests/lib.sh b/scripts/tests/lib.sh new file mode 100755 index 000000000..305eb96d1 --- /dev/null +++ b/scripts/tests/lib.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +function install_iptables_drop_rules() { + sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP + sudo iptables -I FORWARD 1 -s 172.28.0.105 -d 172.28.0.100 -j DROP +} + +function remove_iptables_drop_rules() { + sudo iptables -D FORWARD -s 172.28.0.100 -d 172.28.0.105 -j DROP + sudo iptables -D FORWARD -s 172.28.0.105 -d 172.28.0.100 -j DROP +} + +function client_ping_gateway() { + docker compose exec -it client timeout 60 sh -c 'until ping -W 1 -c 10 172.20.0.100 &>/dev/null; do true; done' +} diff --git a/scripts/tests/relayed-ping-portal-down.sh b/scripts/tests/relayed-ping-portal-down.sh new file mode 100755 index 000000000..8a5a246f8 --- /dev/null +++ b/scripts/tests/relayed-ping-portal-down.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -e + +source "./scripts/tests/lib.sh" + +install_iptables_drop_rules +trap remove_iptables_drop_rules EXIT # Cleanup after us + +client_ping_gateway + +docker compose stop api # Stop portal + +sleep 5 # Wait for client to disconnect + +client_ping_gateway diff --git a/scripts/tests/relayed-ping-portal-restart.sh b/scripts/tests/relayed-ping-portal-restart.sh new file mode 100755 index 000000000..9e2765d77 --- /dev/null +++ b/scripts/tests/relayed-ping-portal-restart.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -e + +source "./scripts/tests/lib.sh" + +install_iptables_drop_rules +trap remove_iptables_drop_rules EXIT # Cleanup after us + +client_ping_gateway + +docker compose restart api # Restart portal + +sleep 5 # Wait for client to reconnect + +client_ping_gateway