firezone/.github/workflows/ci.yml

name: Continuous Integration
on:
  pull_request:
  merge_group:
    types: [checks_requested]
  workflow_call:

# Cancel old workflow runs if new code is pushed
concurrency:
  group: "ci-${{ github.workflow }}-${{ github.ref }}"
  cancel-in-progress: true

jobs:
  elixir:
    uses: ./.github/workflows/_elixir.yml
  rust:
    uses: ./.github/workflows/_rust.yml
  kotlin:
    uses: ./.github/workflows/_kotlin.yml
    secrets: inherit
  swift:
    uses: ./.github/workflows/_swift.yml
    secrets: inherit
  static-analysis:
    uses: ./.github/workflows/_static-analysis.yml
  terraform:
    uses: ./.github/workflows/_terraform.yml
    secrets: inherit
  codeql:
    uses: ./.github/workflows/_codeql.yml
    secrets: inherit

  # We could build these in GCP with Cloud Build, but for now it's
  # less overhead to keep things in GH actions. See work on building these
  # in GCP with Cloud Build: https://github.com/firezone/firezone/pull/2234
  build-images:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        include:
          - image_name: api
            target: runtime
            context: elixir
            build-args: |
              APPLICATION_NAME=api
          - image_name: web
            target: runtime
            context: elixir
            build-args: |
              APPLICATION_NAME=web
          - image_name: gateway
            target: debug
            context: rust
            build-args: |
              PACKAGE=firezone-gateway
          - image_name: relay
            target: debug
            context: rust
            build-args: |
              PACKAGE=firezone-relay
          - image_name: client
            target: debug
            context: rust
            build-args: |
              PACKAGE=firezone-linux-client
          - image_name: elixir
            target: compiler
            context: elixir
            build-args: |
              APPLICATION_NAME=api
    permissions:
      contents: read
      id-token: write
    env:
      # mark:automatic-version
      VERSION: "1.20231001.0"
      APPLICATION_NAME: ${{ matrix.image_name }}
    steps:
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          # We are overriding the default buildkit version being used by Buildx. We need buildkit >= 12.0 and currently BuildX
          # supports v0.11.6 https://github.com/docker/buildx/blob/b8739d74417f86aa8fc9aafb830a8ba656bdef0e/Dockerfile#L9.
          # We should for any updates on buildx and on the setup-buildx-action itself.
          driver-opts: |
            image=moby/buildkit:v0.12.0
      - uses: actions/checkout@v4
      - name: Sanitize github.ref_name
        run: |
          # `ref_name` contains `/` which is not a valid docker image tag
          REF="${{ github.ref_name }}"
          CACHE_TAG="${REF//\//-}"
          echo "CACHE_TAG=$CACHE_TAG" >> "$GITHUB_ENV"
          echo "BRANCH_TAG=$CACHE_TAG" >> "$GITHUB_ENV"
      - uses: ./.github/actions/gcp-docker-login
        id: login
        with:
          project: firezone-staging
      - name: Build Docker Tags
        id: build_docker_tags
        run: |
          set -xe

          TAGS=""

          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
            MAJOR_VERSION="${VERSION%%.*}"
            MAJOR_MINOR_VERSION="${VERSION%.*}"

            TAGS="${TAGS},${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${MAJOR_VERSION}"
            TAGS="${TAGS},${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${MAJOR_MINOR_VERSION}"
            TAGS="${TAGS},${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ env.VERSION }}-${{ github.sha }}"
          fi

          TAGS="${TAGS},${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ env.BRANCH_TAG }}"
          TAGS="${TAGS},${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ github.sha }}"

          echo "::set-output name=tags::${TAGS}"
      - name: Build Docker images
        uses: docker/build-push-action@v5
        with:
          platforms: linux/amd64
          build-args: ${{ matrix.build-args }}
          context: ${{ matrix.context }}/
          cache-from: |
            type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
            type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.image_name }}:main
          # This will write the cache on main even if integration tests fail,
          # but it'll just be corrected on the next successful build.
          cache-to: |
            type=registry,ref=${{steps.login.outputs.registry}}/cache/${{ matrix.image_name}}:${{ env.CACHE_TAG }},mode=max
          file: ${{ matrix.context }}/Dockerfile
          push: true
          target: ${{ matrix.target }}
          tags: ${{ steps.build_docker_tags.outputs.tags }}

  integration-tests:
    needs: build-images
    runs-on: ubuntu-22.04
    permissions:
      contents: read
      id-token: write
      pull-requests: write
    env:
      VERSION: ${{ github.sha }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # TODO
          # - Run control plane components as services
          # - Test clients
          # - Test with different NAT types
          # - Test IPv6
          # - Test end-to-end critical paths
          - test_name: Relayed flow
            artifact_name: relayed
            setup: |
              #  Disallow traffic between gateway and client container
              sudo iptables -I FORWARD 1 -s 172.28.0.100 -d 172.28.0.105 -j DROP
              sudo iptables -I FORWARD 1 -s 172.28.0.105 -d 172.28.0.100 -j DROP
            execute: |
              docker compose exec -it client timeout 60 \
              sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done'
          - test_name: Basic flow
            artifact_name: direct
            setup: echo 'Noop'
            execute: |
              docker compose exec -it client timeout 60 \
              sh -c 'until ping -W 1 -c 1 172.20.0.100 &>/dev/null; do true; done'
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/gcp-docker-login
        id: login
        with:
          project: firezone-staging
      - name: Seed database
        run:
          docker compose run elixir /bin/sh -c 'cd apps/domain && mix ecto.seed'
      - name: Start docker compose in the background
        run: |
          docker compose up -d \
            api \
            web \
            client \
            relay \
            gateway \
            iperf3
      - name: Setup ${{ matrix.test_name }} test
        run: ${{ matrix.setup }}
      - name: Execute ${{ matrix.test_name }} test
        run: ${{ matrix.execute }}
      - name: ${{ matrix.test_name }} Perfomance Test
        id: perfomance-test
        run: |
          set -xe

          # We need to increase the log level to make sure that they don't hold off storm of packets
          # generated by UDP tests. Wire is especially chatty.
          sed -i 's/^\(\s*\)RUST_LOG:.*$/\1RUST_LOG: wire=error,info/' docker-compose.yml
          cat docker-compose.yml | grep RUST_LOG
          docker compose down client gateway relay
          docker compose up -d client gateway relay

          mkdir -p /tmp/iperf3results
          docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 -R -c 172.20.0.110 --json' >> /tmp/iperf3results/tcp_server2client.json
          cat /tmp/iperf3results/tcp_server2client.json | jq -r '"tcp_server2client_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_retransmits=" + (.end.sum_sent.retransmits|tostring)' >> "$GITHUB_OUTPUT"

          docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 -c 172.20.0.110 --json' >> /tmp/iperf3results/tcp_client2server.json
          cat /tmp/iperf3results/tcp_client2server.json | jq -r '"tcp_client2server_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results/tcp_client2server.json | jq -r '"tcp_client2server_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results/tcp_client2server.json | jq -r '"tcp_client2server_sum_sent_retransmits=" + (.end.sum_sent.retransmits|tostring)' >> "$GITHUB_OUTPUT"

          # Note: birtate is reduced to be 250M but what we actually want to test for is 1G once we flesh out some bugs
          docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 -u -b 250M -R -c 172.20.0.110 --json' >> /tmp/iperf3results/udp_server2client.json
          cat /tmp/iperf3results/udp_server2client.json | jq -r '"udp_server2client_sum_bits_per_second=" + (.end.sum.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results/udp_server2client.json | jq -r '"udp_server2client_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results/udp_server2client.json | jq -r '"udp_server2client_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT"

          # Note: birtate is reduced to be 250M but what we actually want to test for is 1G once we flesh out some bugs
          docker compose exec --env RUST_LOG=info -it client /bin/sh -c 'iperf3 -u -b 250M -c 172.20.0.110 --json' >> /tmp/iperf3results/udp_client2server.json
          cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_bits_per_second=" + (.end.sum.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results/udp_client2server.json | jq -r '"udp_client2server_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT"
      - name: Save ${{ matrix.test_name }} Perfomance Test Results
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.artifact_name }}-iperf3results
          path: /tmp/iperf3results
      - name: Download main branch ${{ matrix.test_name }} Perfomance Test Results
        id: download-artifact
        if: ${{ github.event_name == 'pull_request' }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -xe

          REPO="${{ github.repository }}"
          WORKFLOW="cd.yml"
          ARTIFACT_NAME="${{ matrix.artifact_name }}-iperf3results"
          DESTINATION="/tmp/iperf3results-main"

          ARTIFACTS_URL=$(
            gh api \
              -H "Accept: application/vnd.github+json" \
              -H "X-GitHub-Api-Version: 2022-11-28" \
              "/repos/${REPO}/actions/workflows/${WORKFLOW}/runs?event=push&branch=main&status=success&per_page=1" \
              --jq ".workflow_runs[0].artifacts_url"
          )

          DOWNLOAD_URL=$(
            gh api \
              -H "Accept: application/vnd.github+json" \
              -H "X-GitHub-Api-Version: 2022-11-28" \
              "${ARTIFACTS_URL}" \
              --jq '.artifacts[] | select(.name == "'${ARTIFACT_NAME}'") | .archive_download_url'
          )

          set +x
          curl -H "Accept: application/vnd.github+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -L -o "${DESTINATION}.zip" "$DOWNLOAD_URL"

          set -x
          unzip "${DESTINATION}.zip" -d "${DESTINATION}"
          rm "${DESTINATION}.zip"
      - name: "Generate main branch metrics"
        id: main-perfomance-test
        if: ${{ github.event_name == 'pull_request' }}
        run: |
          cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results-main/tcp_server2client.json | jq -r '"tcp_server2client_sum_sent_retransmits=" + (.end.sum_sent.retransmits|tostring)' >> "$GITHUB_OUTPUT"

          cat /tmp/iperf3results-main/tcp_client2server.json | jq -r '"tcp_client2server_sum_received_bits_per_second=" + (.end.sum_received.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results-main/tcp_client2server.json | jq -r '"tcp_client2server_sum_sent_bits_per_second=" + (.end.sum_sent.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results-main/tcp_client2server.json | jq -r '"tcp_client2server_sum_sent_retransmits=" + (.end.sum_sent.retransmits|tostring)' >> "$GITHUB_OUTPUT"

          cat /tmp/iperf3results-main/udp_server2client.json | jq -r '"udp_server2client_sum_bits_per_second=" + (.end.sum.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results-main/udp_server2client.json | jq -r '"udp_server2client_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results-main/udp_server2client.json | jq -r '"udp_server2client_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT"

          cat /tmp/iperf3results-main/udp_client2server.json | jq -r '"udp_client2server_sum_bits_per_second=" + (.end.sum.bits_per_second|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results-main/udp_client2server.json | jq -r '"udp_client2server_sum_jitter_ms=" + (.end.sum.jitter_ms|tostring)' >> "$GITHUB_OUTPUT"
          cat /tmp/iperf3results-main/udp_client2server.json | jq -r '"udp_client2server_sum_lost_percent=" + (.end.sum.lost_percent|tostring)' >> "$GITHUB_OUTPUT"
      - name: Update PR
        uses: actions/github-script@v7
        id: perf-comment
        if: ${{ github.event_name == 'pull_request' }}
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            // 1. Retrieve existing bot comments for the PR
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });

            const botComment = comments.find(comment => {
              return comment.user.type === 'Bot' && comment.body.includes('${{ matrix.test_name }} Perfomance Test Results')
            });

            function humanFileSize(bytes, dp=1) {
              const thresh = 1000;

              if (Math.abs(bytes) < thresh) {
                return bytes + ' B';
              }

              const units = ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
              let u = -1;
              const r = 10**dp;

              do {
                bytes /= thresh;
                ++u;
              } while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1);

              return bytes.toFixed(dp) + ' ' + units[u];
            }

            function getDiffPercents(main, current)
            {
                let diff = -1 * (100 - current / (main / 100));

                if (diff > 0) {
                    return "+" + diff.toFixed(0) + "%";
                } else if (diff == 0) {
                    return "0%";
                } else {
                    return diff.toFixed(0) + "%";
                }
            }

            let tcp_server2client_sum_received_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.tcp_server2client_sum_received_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_server2client_sum_received_bits_per_second }}, ${{ steps.perfomance-test.outputs.tcp_server2client_sum_received_bits_per_second }}) + ')';
            let tcp_server2client_sum_sent_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.tcp_server2client_sum_sent_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_server2client_sum_sent_bits_per_second }}, ${{ steps.perfomance-test.outputs.tcp_server2client_sum_sent_bits_per_second }}) + ')';
            let tcp_server2client_sum_sent_retransmits = ${{ steps.perfomance-test.outputs.tcp_server2client_sum_sent_retransmits }} + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_server2client_sum_sent_retransmits }}, ${{ steps.perfomance-test.outputs.tcp_server2client_sum_sent_retransmits }}) + ')';

            let tcp_client2server_sum_received_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.tcp_client2server_sum_received_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_client2server_sum_received_bits_per_second }}, ${{ steps.perfomance-test.outputs.tcp_client2server_sum_received_bits_per_second }}) + ')';
            let tcp_client2server_sum_sent_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.tcp_client2server_sum_sent_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_client2server_sum_sent_bits_per_second }}, ${{ steps.perfomance-test.outputs.tcp_client2server_sum_sent_bits_per_second }}) + ')';
            let tcp_client2server_sum_sent_retransmits = ${{ steps.perfomance-test.outputs.tcp_client2server_sum_sent_retransmits }} + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.tcp_client2server_sum_sent_retransmits }}, ${{ steps.perfomance-test.outputs.tcp_client2server_sum_sent_retransmits }}) + ')';

            let udp_server2client_sum_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.udp_server2client_sum_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_server2client_sum_bits_per_second }}, ${{ steps.perfomance-test.outputs.udp_server2client_sum_bits_per_second }}) + ')';
            let udp_server2client_sum_jitter_ms = (${{ steps.perfomance-test.outputs.udp_server2client_sum_jitter_ms }}).toFixed(2) + "ms (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_server2client_sum_jitter_ms }}, ${{ steps.perfomance-test.outputs.udp_server2client_sum_jitter_ms }}) + ')';
            let udp_server2client_sum_lost_percent = (${{ steps.perfomance-test.outputs.udp_server2client_sum_lost_percent }}).toFixed(2) + "% (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_server2client_sum_lost_percent }}, ${{ steps.perfomance-test.outputs.udp_server2client_sum_lost_percent }}) + ')';

            let udp_client2server_sum_bits_per_second = humanFileSize(${{ steps.perfomance-test.outputs.udp_client2server_sum_bits_per_second }}) + ' (' + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_client2server_sum_bits_per_second }}, ${{ steps.perfomance-test.outputs.udp_client2server_sum_bits_per_second }}) + ')';
            let udp_client2server_sum_jitter_ms = (${{ steps.perfomance-test.outputs.udp_client2server_sum_jitter_ms }}).toFixed(2) + "ms (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_client2server_sum_jitter_ms }}, ${{ steps.perfomance-test.outputs.udp_client2server_sum_jitter_ms }}) + ')';
            let udp_client2server_sum_lost_percent = (${{ steps.perfomance-test.outputs.udp_client2server_sum_lost_percent }}).toFixed(2) + "% (" + getDiffPercents(${{ steps.main-perfomance-test.outputs.udp_client2server_sum_lost_percent }}, ${{ steps.perfomance-test.outputs.udp_client2server_sum_lost_percent }}) + ')';

            const output = `## ${{ matrix.test_name }} Perfomance Test Results

            ### TCP

            | Direction        | Received/s                                             | Sent/s                                             | Retransmits                                    |
            |------------------|--------------------------------------------------------|----------------------------------------------------|------------------------------------------------|
            | Client to Server | ` + tcp_client2server_sum_received_bits_per_second + ` | ` + tcp_client2server_sum_sent_bits_per_second + ` | ` + tcp_client2server_sum_sent_retransmits + ` |
            | Server to Client | ` + tcp_server2client_sum_received_bits_per_second + ` | ` + tcp_server2client_sum_sent_bits_per_second + ` | ` + tcp_server2client_sum_sent_retransmits + ` |

            ### UDP

            | Direction        | Total/s                                       | Jitter                                  | Lost                                       |
            |------------------|-----------------------------------------------|-----------------------------------------|--------------------------------------------|
            | Client to Server | ` + udp_client2server_sum_bits_per_second + ` | ` + udp_client2server_sum_jitter_ms + ` | ` + udp_server2client_sum_lost_percent + ` |
            | Server to Client | ` + udp_server2client_sum_bits_per_second + ` | ` + udp_server2client_sum_jitter_ms + ` | ` + udp_client2server_sum_lost_percent + ` |

            `;

            // 3. Update previous comment or create new one
            if (botComment) {
              github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: botComment.id,
                body: output
              });
            } else {
              github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: output
              });
            }
      - name: Show Client logs
        if: "!cancelled()"
        run: docker compose logs client
      - name: Show Relay logs
        if: "!cancelled()"
        run: docker compose logs relay
      - name: Show Gateway logs
        if: "!cancelled()"
        run: docker compose logs gateway
      - name: Show API logs
        if: "!cancelled()"
        run: docker compose logs api
      - name: Show httpbin logs
        if: "!cancelled()"
        run: docker compose logs httpbin