Files
firezone/.github/workflows/_build_artifacts.yml
Thomas Eizinger eeadde0c86 ci: bump Ubuntu runners to 24.04 (#10288)
Ubuntu 22.04 is over 3 years old and therefore ships with quite an old
kernel. Our production VMs (for relays) all run Ubuntu 24.04 so it makes
sense to build and test them on the same kernel / OS release. For
consistency reasons, we therefore bump all runners to 24.04.
2025-09-04 02:04:55 +00:00

510 lines
22 KiB
YAML

name: Build Artifacts
run-name: Triggered from ${{ github.event_name }} by ${{ github.actor }}
on:
workflow_call:
inputs:
image_prefix:
description: |
The prefix to prepend to the image name to prevent SHA conflicts.
* Use "debug" to build debug binaries inside debug stage images + with debug tooling installed.
* Use "perf" to build release binaries inside debug stage images + with debug tooling installed.
* Leave blank to build release binaries inside release stage images.
required: false
type: string
sha:
required: false
type: string
default: ${{ github.sha }}
profile:
description: "The Rust profile to build data plane components with"
required: true
type: string
stage:
description: "The stage of the data plane component images to build"
required: true
type: string
outputs:
client_image:
description: "The client image that was built"
value: ${{ jobs.data-plane-linux.outputs.client_image }}
relay_image:
description: "The relay image that was built"
value: ${{ jobs.data-plane-linux.outputs.relay_image }}
gateway_image:
description: "The gateway image that was built"
value: ${{ jobs.data-plane-linux.outputs.gateway_image }}
http_test_server_image:
description: "The http_test_server image that was built"
value: ${{ jobs.data-plane-linux.outputs.http_test_server_image }}
permissions:
# write permission is required to create a github release
contents: write
id-token: write
packages: write
jobs:
control-plane:
name: ${{ matrix.image_name }}
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
include:
- image_name: domain
target: runtime
build-args: |
APPLICATION_NAME=domain
GIT_SHA=${{ inputs.sha }}
- image_name: api
target: runtime
build-args: |
APPLICATION_NAME=api
GIT_SHA=${{ inputs.sha }}
- image_name: web
target: runtime
build-args: |
APPLICATION_NAME=web
GIT_SHA=${{ inputs.sha }}
- image_name: elixir
target: compiler
build-args: |
APPLICATION_NAME=api
GIT_SHA=${{ inputs.sha }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.sha }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- uses: ./.github/actions/ghcr-docker-login
id: login
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Docker meta
id: meta
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
with:
images: ${{ steps.login.outputs.registry }}/firezone/${{matrix.image_name }}
tags: |
type=raw,value=${{ inputs.sha }}
- name: Sanitize github.ref_name
run: |
# `ref_name` contains `/`, '_' or '=' which is not a valid docker image tag
REF="${{ github.ref_name }}"
CACHE_TAG="${REF//[\/_=]/-}"
echo "CACHE_TAG=$CACHE_TAG" >> "$GITHUB_ENV"
# PRs & non-main branches: read-only cache
- name: Build and push control plane images (read-only cache)
if: ${{ github.ref != 'refs/heads/main' }}
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
build-args: ${{ matrix.build-args }}
target: ${{ matrix.target }}
context: elixir
cache-from: |
type=gha,scope=${{ matrix.image_name }}:${{ env.CACHE_TAG }}
type=gha,scope=${{ matrix.image_name }}:main
# no cache-to here -> read-only
push: true
tags: |
${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ inputs.sha }}
${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
# main: read/write cache
- name: Build and push control plane images (read/write cache)
if: ${{ github.ref == 'refs/heads/main' }}
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
build-args: ${{ matrix.build-args }}
target: ${{ matrix.target }}
context: elixir
cache-from: |
type=gha,scope=${{ matrix.image_name }}:${{ env.CACHE_TAG }}
type=gha,scope=${{ matrix.image_name }}:main
cache-to: |
type=gha,scope=${{ matrix.image_name }}:${{ env.CACHE_TAG }},mode=max,ignore-error=true
push: true
tags: |
${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ inputs.sha }}
${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
data-plane-windows:
name: client-windows-${{ matrix.target }}
if: ${{ inputs.image_prefix != 'perf' }} # Perf testing happens only on Linux
runs-on: windows-2022
defaults:
run:
working-directory: rust
strategy:
fail-fast: false
matrix:
# TODO: Add ARM64 support
artifact: [firezone-client-headless-windows]
arch: [x86_64]
target: [x86_64-pc-windows-msvc]
package: [firezone-headless-client]
# mark:next-headless-version
release_name: [headless-client-1.5.3]
# mark:next-headless-version
version: [1.5.3]
env:
ARTIFACT_PATH: ${{ matrix.artifact }}_${{ matrix.version }}_${{ matrix.arch }}.exe
RELEASE_NAME: ${{ matrix.release_name }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.sha }}
- uses: ./.github/actions/setup-rust
with:
targets: ${{ matrix.target }}
sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
- name: Build binaries
shell: bash
run: |
set -xe
if [[ "${{ inputs.profile }}" == "release" ]]; then
PROFILE="--release"
else
PROFILE=""
fi
cargo build $PROFILE -p ${{ matrix.package }} --target ${{ matrix.target }}
mv target/${{ matrix.target }}/${{ inputs.profile }}/${{ matrix.package }}.exe "$ARTIFACT_PATH"
- name: Install AzureSignTool
shell: bash
run: dotnet tool install --global AzureSignTool
- name: Sign the binary
shell: bash
env:
AZURE_KEY_VAULT_URI: ${{ secrets.AZURE_KEY_VAULT_URI }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_CERT_NAME: ${{ secrets.AZURE_CERT_NAME }}
run: ../scripts/build/sign.sh "$ARTIFACT_PATH"
- name: Upload Release Assets
if: ${{ inputs.profile == 'release' && inputs.stage == 'release' && matrix.release_name }}
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ../scripts/upload/github-release.sh
data-plane-linux:
name: ${{ matrix.name.image_name }}-${{ matrix.arch.shortname }}
runs-on: ubuntu-24.04
defaults:
run:
working-directory: rust
strategy:
fail-fast: false
matrix:
# Copy input vars to matrix vars to conditionally exclude them
image_prefix:
- ${{ inputs.image_prefix }}
stage:
- ${{ inputs.stage }}
# Syntax is weird because https://github.com/actions/runner/issues/1512
exclude:
# Devs are either on amd64 or arm64
- { stage: debug, arch: { platform: linux/arm/v7 } }
# Exclude http-test-server from perf image builds
- { image_prefix: perf, name: { package: http-test-server } }
arch:
- target: x86_64-unknown-linux-musl
shortname: x86_64
platform: linux/amd64
install_dependencies: |
sudo apt-get install musl-tools
- target: aarch64-unknown-linux-musl # E.g. AWS Graviton
shortname: aarch64
platform: linux/arm64
install_dependencies: |
# TODO: musl.cc has blocked GitHub actions: https://github.com/orgs/community/discussions/27906
# Find some other way to keep these updated.
if [[ ! -x /tmp/toolchain/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc ]]; then
curl -fsSL https://github.com/firezone/musl-toolchains/releases/download/1/aarch64-linux-musl-cross.tgz -o /tmp/aarch64-linux-musl-cross.tgz
mkdir -p /tmp/toolchain
tar -xzf /tmp/aarch64-linux-musl-cross.tgz -C /tmp/toolchain
fi
CC=/tmp/toolchain/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc
echo "CC_aarch64_unknown_linux_musl=$CC" >> $GITHUB_ENV
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=$CC" >> $GITHUB_ENV
- target: armv7-unknown-linux-musleabihf # E.g. Raspberry Pi
platform: linux/arm/v7
shortname: armv7
install_dependencies: |
# TODO: musl.cc has blocked GitHub actions: https://github.com/orgs/community/discussions/27906
# Find some other way to keep these updated.
if [[ ! -x /tmp/toolchain/arm-linux-musleabihf-cross/bin/arm-linux-musleabihf-gcc ]]; then
curl -fsSL https://github.com/firezone/musl-toolchains/releases/download/1/arm-linux-musleabihf-cross.tgz -o /tmp/arm-linux-musleabihf-cross.tgz
mkdir -p /tmp/toolchain
tar -xzf /tmp/arm-linux-musleabihf-cross.tgz -C /tmp/toolchain
fi
CC=/tmp/toolchain/arm-linux-musleabihf-cross/bin/arm-linux-musleabihf-gcc
echo "CC_armv7_unknown_linux_musleabihf=$CC" >> $GITHUB_ENV
echo "CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABIHF_LINKER=$CC" >> $GITHUB_ENV
name:
- package: firezone-headless-client
artifact: firezone-client-headless-linux
image_name: client
# mark:next-headless-version
release_name: headless-client-1.5.3
# mark:next-headless-version
version: 1.5.3
- package: firezone-relay
artifact: firezone-relay
image_name: relay
- package: firezone-gateway
artifact: firezone-gateway
image_name: gateway
# mark:next-gateway-version
release_name: gateway-1.4.16
# mark:next-gateway-version
version: 1.4.16
- package: http-test-server
artifact: http-test-server
image_name: http-test-server
env:
BINARY_DEST_PATH: ${{ matrix.name.artifact }}_${{ matrix.name.version }}_${{ matrix.arch.shortname }}
SENTRY_ENVIRONMENT: "production"
outputs:
client_image: ${{ steps.image-name.outputs.client_image }}
relay_image: ${{ steps.image-name.outputs.relay_image }}
gateway_image: ${{ steps.image-name.outputs.gateway_image }}
http_test_server_image: ${{ steps.image-name.outputs.http-test-server_image }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.sha }}
- uses: ./.github/actions/ghcr-docker-login
id: login
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- uses: ./.github/actions/setup-rust
with:
targets: ${{ matrix.arch.target }}
sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
- name: Install dependencies
run: ${{ matrix.arch.install_dependencies }}
- uses: taiki-e/install-action@d31232495ad76f47aad66e3501e47780b49f0f3e # v2.57.5
with:
tool: bpf-linker
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Build binaries
run: |
set -xe
if [[ "${{ inputs.profile }}" == "release" ]]; then
PROFILE="--release"
else
PROFILE=""
fi
cargo build $PROFILE -p ${{ matrix.name.package }} --target ${{ matrix.arch.target }}
# Used for Docker images
cp target/${{ matrix.arch.target }}/${{ inputs.profile }}/${{ matrix.name.package }} ${{ matrix.name.package }}
sha256sum ${{ matrix.name.package }} > ${{ matrix.name.package }}.sha256sum.txt
# For pushing built images to Google Cloud Storage
- uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
if: ${{ inputs.profile == 'release' && matrix.stage == 'release' && matrix.name.artifact == 'firezone-relay' }}
with:
token_format: access_token
workload_identity_provider: "projects/85623168602/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
service_account: "github-account@firezone-staging.iam.gserviceaccount.com"
export_environment_variables: true
create_credentials_file: true
- name: Copy relay to Cloud Storage
if: ${{ inputs.profile == 'release' && matrix.stage == 'release' && matrix.name.artifact == 'firezone-relay' }}
run: |
set -e
gcloud storage cp \
"${{ matrix.name.package }}" \
gs://firezone-staging-artifacts/${{ matrix.name.image_name }}/${{ inputs.sha }}/${{ matrix.arch.shortname }}
gcloud storage cp \
"${{ matrix.name.package }}".sha256sum.txt \
gs://firezone-staging-artifacts/${{ matrix.name.image_name }}/${{ inputs.sha }}/${{ matrix.arch.shortname }}.sha256sum.txt
az storage blob upload \
--container-name artifacts \
--name "${{ matrix.name.image_name }}/${{ inputs.sha }}/${{ matrix.arch.shortname }}" \
--file "${{ matrix.name.package }}" \
--overwrite true \
--no-progress \
--connection-string "${{ secrets.AZURERM_ARTIFACTS_CONNECTION_STRING }}"
az storage blob upload \
--container-name artifacts \
--name "${{ matrix.name.image_name }}/${{ inputs.sha }}/${{ matrix.arch.shortname }}.sha256sum.txt" \
--file "${{ matrix.name.package }}.sha256sum.txt" \
--overwrite true \
--no-progress \
--connection-string "${{ secrets.AZURERM_ARTIFACTS_CONNECTION_STRING }}"
- name: Upload Release Assets
if: ${{ inputs.profile == 'release' && matrix.stage == 'release' && matrix.name.release_name }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -xe
# Only clobber existing release assets if the release is a draft
is_draft=$(gh release view ${{ matrix.name.release_name }} --json isDraft --jq '.isDraft' | tr -d '\n')
if [[ "$is_draft" == "true" ]]; then
clobber="--clobber"
else
clobber=""
fi
# Used for release artifact
cp ${{ matrix.name.package }} "$BINARY_DEST_PATH"
cp ${{ matrix.name.package }}.sha256sum.txt "$BINARY_DEST_PATH".sha256sum.txt
gh release upload ${{ matrix.name.release_name }} \
"$BINARY_DEST_PATH" \
"$BINARY_DEST_PATH".sha256sum.txt \
"$clobber" \
--repo ${{ github.repository }}
- name: Set up QEMU
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Docker meta
id: meta
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
with:
images: ${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }}
# We only version client and gateway
tags: |
type=raw,value={{branch}}
type=raw,value=${{ inputs.sha }}
- name: Sanitize github.ref_name
run: |
# `ref_name` contains `/`, '_' or '=' which is not a valid docker image tag
REF="${{ github.ref_name }}"
CACHE_TAG="${REF//[\/_=]/-}"
echo "CACHE_TAG=$CACHE_TAG" >> "$GITHUB_ENV"
# PRs & non-main branches: read-only cache
- name: Build Docker images (read-only cache)
if: ${{ github.ref != 'refs/heads/main' }}
id: build_ro
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
platforms: ${{ matrix.arch.platform }}
build-args: |
PACKAGE=${{ matrix.name.package }}
TARGET=${{ matrix.arch.target }}
context: rust
cache-from: |
type=gha,scope=${{ matrix.name.image_name }}:${{ env.CACHE_TAG }}
type=gha,scope=${{ matrix.name.image_name }}:main
# no cache-to -> read-only
target: ${{ matrix.stage }}
outputs: type=image,name=${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }},push-by-digest=true,name-canonical=true,push=true
# main: read/write cache
- name: Build Docker images (read/write cache)
if: ${{ github.ref == 'refs/heads/main' }}
id: build_rw
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
platforms: ${{ matrix.arch.platform }}
build-args: |
PACKAGE=${{ matrix.name.package }}
TARGET=${{ matrix.arch.target }}
context: rust
cache-from: |
type=gha,scope=${{ matrix.name.image_name }}:${{ env.CACHE_TAG }}
type=gha,scope=${{ matrix.name.image_name }}:main
cache-to: |
type=gha,scope=${{ matrix.name.image_name }}:${{ env.CACHE_TAG }},mode=max,ignore-error=true
target: ${{ matrix.stage }}
outputs: type=image,name=${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }},push-by-digest=true,name-canonical=true,push=true
- name: Export digest
run: |
mkdir -p /tmp/digests/${{ matrix.name.image_name }}
digest="${{ github.ref == 'refs/heads/main' && steps.build_rw.outputs.digest || steps.build_ro.outputs.digest }}"
touch "/tmp/digests/${{ matrix.name.image_name }}/${digest#sha256:}"
- name: Upload digest artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
overwrite: true
name: ${{ matrix.image_prefix && format('{0}-', matrix.image_prefix) || '' }}${{ matrix.name.image_name }}-${{ inputs.sha }}-digest-${{ matrix.arch.shortname }}
path: /tmp/digests/${{ matrix.name.image_name }}
if-no-files-found: error
retention-days: 1
- name: Output image name
id: image-name
run: echo "${{ matrix.name.image_name }}_image=${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }}" >> "$GITHUB_OUTPUT"
merge-docker-artifacts:
name: merge-${{ matrix.image.name }}
needs: data-plane-linux
if: ${{ always() }}
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
# Copy input vars to matrix vars to conditionally exclude them
image_prefix:
- ${{ inputs.image_prefix }}
# Exclude http-test-server from perf image builds
exclude:
- { image_prefix: perf, image: { name: http-test-server } }
image:
- name: relay
- name: gateway
# mark:next-gateway-version
version: 1.4.16
- name: client
# mark:next-client-version
version: 1.0.6
- name: http-test-server
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.sha }}
- uses: ./.github/actions/ghcr-docker-login
id: login
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Download digests
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
pattern: ${{ matrix.image_prefix && format('{0}-', matrix.image_prefix) || '' }}${{ matrix.image.name }}-${{ inputs.sha }}-digest-*
merge-multiple: true
path: /tmp/digests/${{ matrix.image.name }}
- name: Display structure of downloaded artifacts
run: ls -R /tmp/digests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Docker meta
id: meta
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
with:
images: ${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.image.name }}
tags: |
type=raw,value={{branch}}
type=raw,value=${{ inputs.sha }}
- name: Create manifest list and push
working-directory: /tmp/digests/${{ matrix.image.name }}
run: |
tags=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")
sources=$(printf '${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.image.name }}@sha256:%s ' *)
echo "Tags: $tags"
echo "Sources: $sources"
# shellcheck disable=SC2086 # $tags and $sources must be split by whitespace
docker buildx imagetools create $tags $sources
docker buildx imagetools inspect "${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.image.name }}"