mirror of
https://github.com/outbackdingo/firezone.git
synced 2026-01-27 18:18:55 +00:00
fix(ci): Ensure integration-tests allow for at least 30 seconds to establish a connection (#3676)
So the cause of the flaky tests is that they aren't waiting long enough for a connection to be established. Both the test in #3666 and the `iperf` tests have a timeout of 10 seconds. Connections _should_ be established **very quickly** in CI. However, I have a few guesses as to why they might not be, essentially causing us to have to wait for a timeout to re-initiate a connection request: - Packets arrive out of order or too quickly for the WireGuard state machine to establish a handshake. - Too many ICE candidates gathered (the gateway has 3 interfaces) This PR: - Refactors the iperf tests to be a little easier to maintain - Ensures `integration-tests` run for at least 30 seconds before timing out In any case, we can debug / optimize this further after snownet is merged, which might just solve the problem completely.
This commit is contained in:
Jamil
152
.github/workflows/ci.yml
vendored
152
.github/workflows/ci.yml
vendored
@@ -243,13 +243,11 @@ jobs:
|
||||
include:
|
||||
- test_name: direct-perf
|
||||
setup: echo 'Noop'
|
||||
execute: echo 'Noop'
|
||||
- test_name: relayed-perf
|
||||
setup: |
|
||||
# Disallow traffic between gateway and client container
|
||||
sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP
|
||||
sudo iptables -I FORWARD 1 -s 172.28.0.105 -d 172.28.0.100 -j DROP
|
||||
execute: echo 'Noop'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -277,41 +275,15 @@ jobs:
|
||||
|
||||
- name: 'Setup test: ${{ matrix.test_name }}'
|
||||
run: ${{ matrix.setup }}
|
||||
- name: 'Execute test: ${{ matrix.test_name }}'
|
||||
run: ${{ matrix.execute }}
|
||||
- name: 'Performance test: ${{ matrix.test_name }}'
|
||||
id: perfomance-test
|
||||
id: performance-test
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -xe
|
||||
|
||||
mkdir -p /tmp/iperf3results
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 -O 1 -R -c 172.20.0.110 --json' >> /tmp/iperf3results/tcp_server2client.json
|
||||
cat /tmp/iperf3results/tcp_server2client.json | jq -r '"tcp_server2client_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_retransmits=" + (.end.sum_sent.retransmits|tostring)' >> "$GITHUB_OUTPUT"
|
||||
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 -O 1 -c 172.20.0.110 --json' >> /tmp/iperf3results/tcp_client2server.json
|
||||
cat /tmp/iperf3results/tcp_client2server.json | jq -r '"tcp_client2server_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results/tcp_client2server.json | jq -r '"tcp_client2server_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results/tcp_client2server.json | jq -r '"tcp_client2server_sum_sent_retransmits=" + (.end.sum_sent.retransmits|tostring)' >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Note: birtate is reduced to be 250M but what we actually want to test for is 1G once we flesh out some bugs
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 -O 1 -u -b 250M -R -c 172.20.0.110 --json' >> /tmp/iperf3results/udp_server2client.json
|
||||
cat /tmp/iperf3results/udp_server2client.json | jq -r '"udp_server2client_sum_bits_per_second=" + (.end.sum.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results/udp_server2client.json | jq -r '"udp_server2client_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results/udp_server2client.json | jq -r '"udp_server2client_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Note: birtate is reduced to be 250M but what we actually want to test for is 1G once we flesh out some bugs
|
||||
docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 -O 1 -u -b 250M -c 172.20.0.110 --json' >> /tmp/iperf3results/udp_client2server.json
|
||||
cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_bits_per_second=" + (.end.sum.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT"
|
||||
run: ./scripts/tests/perf/test.sh
|
||||
- name: 'Save performance test results: ${{ matrix.test_name }}'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: '${{ matrix.test_name }}-iperf3results'
|
||||
path: /tmp/iperf3results
|
||||
path: ./iperf3results
|
||||
- name: 'Download main branch performance test results: ${{ matrix.test_name }}'
|
||||
id: download-artifact
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
@@ -323,7 +295,7 @@ jobs:
|
||||
REPO="${{ github.repository }}"
|
||||
WORKFLOW="cd.yml"
|
||||
ARTIFACT_NAME="${{ matrix.test_name }}-iperf3results"
|
||||
DESTINATION="/tmp/iperf3results-main"
|
||||
DESTINATION="./iperf3results-main"
|
||||
|
||||
ARTIFACTS_URL=$(
|
||||
gh api \
|
||||
@@ -347,125 +319,15 @@ jobs:
|
||||
set -x
|
||||
unzip "${DESTINATION}.zip" -d "${DESTINATION}"
|
||||
rm "${DESTINATION}.zip"
|
||||
- name: "Generate main branch metrics"
|
||||
id: main-perfomance-test
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: |
|
||||
cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_retransmits=" + (.end.sum_sent.retransmits|tostring)' >> "$GITHUB_OUTPUT"
|
||||
|
||||
cat /tmp/iperf3results-main/tcp_client2server.json | jq -r '"tcp_client2server_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results-main/tcp_client2server.json | jq -r '"tcp_client2server_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results-main/tcp_client2server.json | jq -r '"tcp_client2server_sum_sent_retransmits=" + (.end.sum_sent.retransmits|tostring)' >> "$GITHUB_OUTPUT"
|
||||
|
||||
cat /tmp/iperf3results-main/udp_server2client.json | jq -r '"udp_server2client_sum_bits_per_second=" + (.end.sum.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results-main/udp_server2client.json | jq -r '"udp_server2client_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results-main/udp_server2client.json | jq -r '"udp_server2client_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT"
|
||||
|
||||
cat /tmp/iperf3results-main/udp_client2server.json | jq -r '"udp_client2server_sum_bits_per_second=" + (.end.sum.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results-main/udp_client2server.json | jq -r '"udp_client2server_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/iperf3results-main/udp_client2server.json | jq -r '"udp_client2server_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT"
|
||||
- name: Update PR
|
||||
- name: Update PR with results
|
||||
uses: actions/github-script@v7
|
||||
id: perf-comment
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
// 1. Retrieve existing bot comments for the PR
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
|
||||
const botComment = comments.find(comment => {
|
||||
return comment.user.type === 'Bot' && comment.body.includes('${{ matrix.test_name }}')
|
||||
});
|
||||
|
||||
function humanFileSize(bytes, dp=1) {
|
||||
const thresh = 1000;
|
||||
|
||||
if (Math.abs(bytes) < thresh) {
|
||||
return bytes + ' B';
|
||||
}
|
||||
|
||||
const units = ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
|
||||
let u = -1;
|
||||
const r = 10**dp;
|
||||
|
||||
do {
|
||||
bytes /= thresh;
|
||||
++u;
|
||||
} while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1);
|
||||
|
||||
return bytes.toFixed(dp) + ' ' + units[u];
|
||||
}
|
||||
|
||||
function getDiffPercents(main, current)
|
||||
{
|
||||
let diff = -1 * (100 - current / (main / 100));
|
||||
|
||||
if (diff > 0) {
|
||||
return "+" + diff.toFixed(0) + "%";
|
||||
} else if (diff == 0) {
|
||||
return "0%";
|
||||
} else {
|
||||
return diff.toFixed(0) + "%";
|
||||
}
|
||||
}
|
||||
|
||||
let tcp_server2client_sum_received_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.tcp_server2client_sum_received_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_server2client_sum_received_bits_per_second }}, ${{ steps.perfomance-test.outputs.tcp_server2client_sum_received_bits_per_second }}) + ')';
|
||||
let tcp_server2client_sum_sent_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.tcp_server2client_sum_sent_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_server2client_sum_sent_bits_per_second }}, ${{ steps.perfomance-test.outputs.tcp_server2client_sum_sent_bits_per_second }}) + ')';
|
||||
let tcp_server2client_sum_sent_retransmits = ${{ steps.perfomance-test.outputs.tcp_server2client_sum_sent_retransmits }} + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_server2client_sum_sent_retransmits }}, ${{ steps.perfomance-test.outputs.tcp_server2client_sum_sent_retransmits }}) + ')';
|
||||
|
||||
let tcp_client2server_sum_received_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.tcp_client2server_sum_received_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_client2server_sum_received_bits_per_second }}, ${{ steps.perfomance-test.outputs.tcp_client2server_sum_received_bits_per_second }}) + ')';
|
||||
let tcp_client2server_sum_sent_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.tcp_client2server_sum_sent_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_client2server_sum_sent_bits_per_second }}, ${{ steps.perfomance-test.outputs.tcp_client2server_sum_sent_bits_per_second }}) + ')';
|
||||
let tcp_client2server_sum_sent_retransmits = ${{ steps.perfomance-test.outputs.tcp_client2server_sum_sent_retransmits }} + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_client2server_sum_sent_retransmits }}, ${{ steps.perfomance-test.outputs.tcp_client2server_sum_sent_retransmits }}) + ')';
|
||||
|
||||
let udp_server2client_sum_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.udp_server2client_sum_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_server2client_sum_bits_per_second }}, ${{ steps.perfomance-test.outputs.udp_server2client_sum_bits_per_second }}) + ')';
|
||||
let udp_server2client_sum_jitter_ms = (${{ steps.perfomance-test.outputs.udp_server2client_sum_jitter_ms }}).toFixed(2) + "ms (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_server2client_sum_jitter_ms }}, ${{ steps.perfomance-test.outputs.udp_server2client_sum_jitter_ms }}) + ')';
|
||||
let udp_server2client_sum_lost_percent = (${{ steps.perfomance-test.outputs.udp_server2client_sum_lost_percent }}).toFixed(2) + "% (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_server2client_sum_lost_percent }}, ${{ steps.perfomance-test.outputs.udp_server2client_sum_lost_percent }}) + ')';
|
||||
|
||||
let udp_client2server_sum_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.udp_client2server_sum_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_client2server_sum_bits_per_second }}, ${{ steps.perfomance-test.outputs.udp_client2server_sum_bits_per_second }}) + ')';
|
||||
let udp_client2server_sum_jitter_ms = (${{ steps.perfomance-test.outputs.udp_client2server_sum_jitter_ms }}).toFixed(2) + "ms (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_client2server_sum_jitter_ms }}, ${{ steps.perfomance-test.outputs.udp_client2server_sum_jitter_ms }}) + ')';
|
||||
let udp_client2server_sum_lost_percent = (${{ steps.perfomance-test.outputs.udp_client2server_sum_lost_percent }}).toFixed(2) + "% (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_client2server_sum_lost_percent }}, ${{ steps.perfomance-test.outputs.udp_client2server_sum_lost_percent }}) + ')';
|
||||
|
||||
const output = `## Performance Test Results: ${{ matrix.test_name }}
|
||||
|
||||
### TCP
|
||||
|
||||
| Direction | Received/s | Sent/s | Retransmits |
|
||||
|------------------|--------------------------------------------------------|----------------------------------------------------|------------------------------------------------|
|
||||
| Client to Server | ` + tcp_client2server_sum_received_bits_per_second + ` | ` + tcp_client2server_sum_sent_bits_per_second + ` | ` + tcp_client2server_sum_sent_retransmits + ` |
|
||||
| Server to Client | ` + tcp_server2client_sum_received_bits_per_second + ` | ` + tcp_server2client_sum_sent_bits_per_second + ` | ` + tcp_server2client_sum_sent_retransmits + ` |
|
||||
|
||||
### UDP
|
||||
|
||||
| Direction | Total/s | Jitter | Lost |
|
||||
|------------------|-----------------------------------------------|-----------------------------------------|--------------------------------------------|
|
||||
| Client to Server | ` + udp_client2server_sum_bits_per_second + ` | ` + udp_client2server_sum_jitter_ms + ` | ` + udp_server2client_sum_lost_percent + ` |
|
||||
| Server to Client | ` + udp_server2client_sum_bits_per_second + ` | ` + udp_server2client_sum_jitter_ms + ` | ` + udp_client2server_sum_lost_percent + ` |
|
||||
|
||||
`;
|
||||
|
||||
// 3. Update previous comment or create new one
|
||||
if (botComment) {
|
||||
github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: botComment.id,
|
||||
body: output
|
||||
});
|
||||
} else {
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: output
|
||||
});
|
||||
}
|
||||
const { script } = require('./scripts/tests/perf/results.js');
|
||||
script(github, context, '${{ matrix.test_name }}');
|
||||
- name: Show Client logs
|
||||
if: "!cancelled()"
|
||||
run: docker compose logs client
|
||||
|
||||
Reference in New Issue
Block a user