diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bd5e7a8e..6d33e6755 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,6 +168,7 @@ jobs: timeout 600 docker compose -f rust/snownet-tests/${{ matrix.file }} up --exit-code-from dialer --abort-on-container-exit integration-tests: + name: Integration Test ${{ matrix.test_name }} needs: build-images runs-on: ubuntu-22.04 permissions: @@ -181,13 +182,13 @@ jobs: matrix: include: # Define new integration tests here. These are executed in parallel in our CI. - # If you define the `perf_test_artifact_name` key, the throughput test will + # If you set `perf_test` to true, the throughput test will # be activated and performance results will be uploaded as artifacts. - # If you leave `perf_test_artifact_name` blank, the performance test will + # If you leave `perf_test` blank or set it to false, the performance test will # be skipped. # Basic connectivity tests - - test_name: Client can ping a resource after portal restarts (Direct) + - test_name: direct-ping-portal-restart setup: echo 'Noop' execute: | docker compose exec -it client timeout 60 \ @@ -202,7 +203,7 @@ jobs: # Ping again docker compose exec -it client timeout 60 \ sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - test_name: Client can ping a resource after portal restarts (Relayed) + - test_name: relayed-ping-portal-restart setup: | # Disallow traffic between gateway and client container sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP @@ -220,7 +221,7 @@ jobs: # Ping again docker compose exec -it client timeout 60 \ sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - test_name: Client can ping a resource when portal is down (Direct) + - test_name: direct-ping-portal-down setup: echo 'Noop' execute: | docker compose exec -it client timeout 60 \ @@ -235,7 +236,7 @@ jobs: # Ping again docker compose exec -it client timeout 60 \ sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - test_name: Client can ping a resource after portal restarts (Relayed) + - test_name: relayed-ping-portal-down setup: | # Disallow traffic between gateway and client container sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP @@ -253,7 +254,7 @@ jobs: # Ping again docker compose exec -it client timeout 60 \ sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - test_name: Client can ping a connected resource when api and relay is down (Direct) + - test_name: direct-ping-portal-relay-down setup: echo 'Noop' execute: | docker compose exec -it client timeout 60 \ @@ -270,15 +271,17 @@ jobs: sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' # Performance Tests - - test_name: Client can ping a resource (Direct) - pert_test_artifact_name: direct + - test_name: direct-perf + # Set this to enable performance testing for this test + perf_test: true setup: echo 'Noop' execute: | # Establish a channel docker compose exec -it client timeout 60 \ sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done' - - test_name: Client can ping a resource (Relayed) - pert_test_artifact_name: relayed + - test_name: relayed-perf + # Set this to enable performance testing for this test + perf_test: true setup: | # Disallow traffic between gateway and client container sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP @@ -307,12 +310,12 @@ jobs: relay \ gateway \ iperf3 - - name: Setup ${{ matrix.test_name }} test + - name: 'Setup test: ${{ matrix.test_name }}' run: ${{ matrix.setup }} - - name: Execute ${{ matrix.test_name }} test + - name: 'Execute test: ${{ matrix.test_name }}' run: ${{ matrix.execute }} - - name: ${{ matrix.test_name }} Performance Test - if: ${{ matrix.perf_test_artifact_name }} + - name: 'Performance test: ${{ matrix.test_name }}' + if: matrix.perf_test id: perfomance-test timeout-minutes: 5 run: | @@ -352,15 +355,15 @@ jobs: cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_bits_per_second=" + (.end.sum.bits_per_second|tostring)' >> "$GITHUB_OUTPUT" cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT" cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT" - - name: Save ${{ matrix.test_name }} Performance Test Results - if: ${{ matrix.perf_test_artifact_name }} + - name: 'Save performance test results: ${{ matrix.test_name }}' + if: matrix.perf_test uses: actions/upload-artifact@v4 with: - name: ${{ matrix.perf_test_artifact_name }}-iperf3results + name: '${{ matrix.test_name }}-iperf3results' path: /tmp/iperf3results - - name: Download main branch ${{ matrix.test_name }} Performance Test Results + - name: 'Download main branch performance test results: ${{ matrix.test_name }}' id: download-artifact - if: ${{ github.event_name == 'pull_request' && matrix.perf_test_artifact_name }} + if: ${{ github.event_name == 'pull_request' && matrix.perf_test }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -368,7 +371,7 @@ jobs: REPO="${{ github.repository }}" WORKFLOW="cd.yml" - ARTIFACT_NAME="${{ matrix.perf_test_artifact_name }}-iperf3results" + ARTIFACT_NAME="${{ matrix.test_name }}-iperf3results" DESTINATION="/tmp/iperf3results-main" ARTIFACTS_URL=$( @@ -395,7 +398,7 @@ jobs: rm "${DESTINATION}.zip" - name: "Generate main branch metrics" id: main-perfomance-test - if: ${{ github.event_name == 'pull_request' && matrix.perf_test_artifact_name }} + if: ${{ github.event_name == 'pull_request' && matrix.perf_test }} run: | cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT" cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT" @@ -415,7 +418,7 @@ jobs: - name: Update PR uses: actions/github-script@v7 id: perf-comment - if: ${{ github.event_name == 'pull_request' && matrix.perf_test_artifact_name }} + if: ${{ github.event_name == 'pull_request' && matrix.perf_test }} with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -427,7 +430,7 @@ jobs: }); const botComment = comments.find(comment => { - return comment.user.type === 'Bot' && comment.body.includes('${{ matrix.test_name }} Performance Test Results') + return comment.user.type === 'Bot' && comment.body.includes('${{ matrix.test_name }}') }); function humanFileSize(bytes, dp=1) { @@ -478,7 +481,7 @@ jobs: let udp_client2server_sum_jitter_ms = (${{ steps.perfomance-test.outputs.udp_client2server_sum_jitter_ms }}).toFixed(2) + "ms (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_client2server_sum_jitter_ms }}, ${{ steps.perfomance-test.outputs.udp_client2server_sum_jitter_ms }}) + ')'; let udp_client2server_sum_lost_percent = (${{ steps.perfomance-test.outputs.udp_client2server_sum_lost_percent }}).toFixed(2) + "% (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_client2server_sum_lost_percent }}, ${{ steps.perfomance-test.outputs.udp_client2server_sum_lost_percent }}) + ')'; - const output = `## ${{ matrix.test_name }} Performance Test Results + const output = `## Performance Test Results: ${{ matrix.test_name }} ### TCP