Files
firezone/.github/workflows/_build_artifacts.yml
2025-02-11 07:22:27 -08:00

469 lines
20 KiB
YAML

name: Build Artifacts
run-name: Triggered from ${{ github.event_name }} by ${{ github.actor }}
on:
workflow_call:
inputs:
image_prefix:
description: |
The prefix to prepend to the image name to prevent SHA conflicts.
* Use "debug" to build debug binaries inside debug stage images + with debug tooling installed.
* Use "perf" to build release binaries inside debug stage images + with debug tooling installed.
* Leave blank to build release binaries inside release stage images.
required: false
type: string
sha:
required: false
type: string
default: ${{ github.sha }}
profile:
description: "The Rust profile to build data plane components with"
required: true
type: string
stage:
description: "The stage of the data plane component images to build"
required: true
type: string
outputs:
client_image:
description: "The client image that was built"
value: ${{ jobs.data-plane-linux.outputs.client_image }}
relay_image:
description: "The relay image that was built"
value: ${{ jobs.data-plane-linux.outputs.relay_image }}
gateway_image:
description: "The gateway image that was built"
value: ${{ jobs.data-plane-linux.outputs.gateway_image }}
http_test_server_image:
description: "The http_test_server image that was built"
value: ${{ jobs.data-plane-linux.outputs.http_test_server_image }}
permissions:
# write permission is required to create a github release
contents: write
id-token: write
jobs:
control-plane:
name: ${{ matrix.image_name }}
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
include:
- image_name: domain
target: runtime
build-args: |
APPLICATION_NAME=domain
GIT_SHA=${{ inputs.sha }}
- image_name: api
target: runtime
build-args: |
APPLICATION_NAME=api
GIT_SHA=${{ inputs.sha }}
- image_name: web
target: runtime
build-args: |
APPLICATION_NAME=web
GIT_SHA=${{ inputs.sha }}
- image_name: elixir
target: compiler
build-args: |
APPLICATION_NAME=api
GIT_SHA=${{ inputs.sha }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.sha }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
with:
# We are overriding the default buildkit version being used by Buildx. We need buildkit >= 12.0 and currently BuildX
# supports v0.11.6 https://github.com/docker/buildx/blob/b8739d74417f86aa8fc9aafb830a8ba656bdef0e/Dockerfile#L9.
# We should for any updates on buildx and on the setup-buildx-action itself.
driver-opts: |
image=moby/buildkit:v0.15.1
- uses: ./.github/actions/gcp-docker-login
id: login
with:
project: firezone-staging
- name: Docker meta
id: meta
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
with:
images: ${{ steps.login.outputs.registry }}/firezone/${{matrix.image_name }}
tags: |
type=raw,value=${{ inputs.sha }}
- name: Sanitize github.ref_name
run: |
# `ref_name` contains `/`, '_' or '=' which is not a valid docker image tag
REF="${{ github.ref_name }}"
CACHE_TAG="${REF//[\/_=]/-}"
echo "CACHE_TAG=$CACHE_TAG" >> "$GITHUB_ENV"
- name: Build and push control plane images
id: build
uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
with:
build-args: ${{ matrix.build-args }}
target: ${{ matrix.target }}
context: elixir
cache-from: |
type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.image_name }}:main
cache-to: |
type=registry,ref=${{steps.login.outputs.registry}}/cache/${{ matrix.image_name}}:${{ env.CACHE_TAG }},mode=max
push: true
tags: |
${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ inputs.sha }}
${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
data-plane-windows:
name: client-windows-${{ matrix.target }}
runs-on: windows-2019
defaults:
run:
working-directory: rust
strategy:
fail-fast: false
matrix:
# TODO: Add ARM64 support
artifact: [firezone-client-headless-windows]
arch: [x86_64]
target: [x86_64-pc-windows-msvc]
package: [firezone-headless-client]
# mark:next-headless-version
release_name: [headless-client-1.4.4]
# mark:next-headless-version
version: [1.4.4]
env:
ARTIFACT_PATH: ${{ matrix.artifact }}_${{ matrix.version }}_${{ matrix.arch }}.exe
RELEASE_NAME: ${{ matrix.release_name }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.sha }}
- uses: ./.github/actions/setup-rust
with:
targets: ${{ matrix.target }}
- name: Build binaries
shell: bash
run: |
set -xe
if [[ "${{ inputs.profile }}" == "release" ]]; then
PROFILE="--release"
else
PROFILE=""
fi
cargo build $PROFILE -p ${{ matrix.package }} --target ${{ matrix.target }}
mv target/${{ matrix.target }}/${{ inputs.profile }}/${{ matrix.package }}.exe $ARTIFACT_PATH
- name: Install AzureSignTool
shell: bash
# AzureSignTool >= 5 needs .NET 8. windows-2019 runner only has .NET 7.
run: dotnet tool install --global AzureSignTool --version 4.0.1
- name: Sign the binary
shell: bash
env:
AZURE_KEY_VAULT_URI: ${{ secrets.AZURE_KEY_VAULT_URI }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_CERT_NAME: ${{ secrets.AZURE_CERT_NAME }}
run: ../scripts/build/sign.sh $ARTIFACT_PATH
- name: Upload Release Assets
if: ${{ inputs.profile == 'release' && inputs.stage == 'release' && matrix.release_name }}
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ../scripts/upload/github-release.sh
data-plane-linux:
name: ${{ matrix.name.image_name }}-${{ matrix.arch.shortname }}
runs-on: ubuntu-22.04
defaults:
run:
working-directory: rust
strategy:
fail-fast: false
matrix:
# Copy input vars to matrix vars to conditionally exclude them
image_prefix:
- ${{ inputs.image_prefix }}
stage:
- ${{ inputs.stage }}
# Syntax is weird because https://github.com/actions/runner/issues/1512
exclude:
# Exclude debug builds for non-amd64 targets since they won't be used.
- { stage: debug, arch: { platform: linux/arm/v7 } }
- { stage: debug, arch: { platform: linux/arm64 } }
# Exclude http-test-server from perf image builds
- { image_prefix: perf, name: { package: http-test-server } }
arch:
- target: x86_64-unknown-linux-musl
shortname: x86_64
platform: linux/amd64
- target: aarch64-unknown-linux-musl # E.g. AWS Graviton
shortname: aarch64
platform: linux/arm64
- target: armv7-unknown-linux-musleabihf # E.g. Raspberry Pi
platform: linux/arm/v7
shortname: armv7
name:
- package: firezone-headless-client
artifact: firezone-client-headless-linux
image_name: client
# mark:next-headless-version
release_name: headless-client-1.4.4
# mark:next-headless-version
version: 1.4.4
- package: firezone-relay
artifact: firezone-relay
image_name: relay
- package: firezone-gateway
artifact: firezone-gateway
image_name: gateway
# mark:next-gateway-version
release_name: gateway-1.4.5
# mark:next-gateway-version
version: 1.4.5
- package: http-test-server
artifact: http-test-server
image_name: http-test-server
env:
BINARY_DEST_PATH: ${{ matrix.name.artifact }}_${{ matrix.name.version }}_${{ matrix.arch.shortname }}
SENTRY_ENVIRONMENT: "production"
outputs:
client_image: ${{ steps.image-name.outputs.client_image }}
relay_image: ${{ steps.image-name.outputs.relay_image }}
gateway_image: ${{ steps.image-name.outputs.gateway_image }}
http_test_server_image: ${{ steps.image-name.outputs.http-test-server_image }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.sha }}
- uses: ./.github/actions/setup-rust
with:
targets: ${{ matrix.arch.target }}
# Cross doesn't support scccache without a lot of work
cache_backend: github
# Cache needs to be scoped per OS version and target since cross seems to clobber the cache
key: ubuntu-22.04-${{ matrix.arch.target }}
- uses: taiki-e/install-action@65835784ace7c5a78745b6dbeb9bf85346430497 # v2.47.32
with:
tool: cross
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Build binaries
run: |
set -xe
if [[ "${{ inputs.profile }}" == "release" ]]; then
PROFILE="--release"
else
PROFILE=""
fi
cross build $PROFILE -p ${{ matrix.name.package }} --target ${{ matrix.arch.target }}
# Used for Docker images
cp target/${{ matrix.arch.target }}/${{ inputs.profile }}/${{ matrix.name.package }} ${{ matrix.name.package }}
- name: Upload Release Assets
if: ${{ inputs.profile == 'release' && matrix.stage == 'release' && matrix.name.release_name }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -xe
# Only clobber existing release assets if the release is a draft
is_draft=$(gh release view ${{ matrix.name.release_name }} --json isDraft --jq '.isDraft' | tr -d '\n')
if [[ "$is_draft" == "true" ]]; then
clobber="--clobber"
else
clobber=""
fi
# Used for release artifact
cp target/${{ matrix.arch.target }}/${{ inputs.profile }}/${{ matrix.name.package }} $BINARY_DEST_PATH
sha256sum $BINARY_DEST_PATH > $BINARY_DEST_PATH.sha256sum.txt
gh release upload ${{ matrix.name.release_name }} \
${{ env.BINARY_DEST_PATH }} \
${{ env.BINARY_DEST_PATH }}.sha256sum.txt \
$clobber \
--repo ${{ github.repository }}
- name: Authenticate to Google Cloud
id: auth
if: ${{ inputs.profile == 'release' && matrix.stage == 'release' && contains(fromJSON('["gateway", "client"]'), matrix.name.image_name) }}
uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
with:
workload_identity_provider: "projects/397012414171/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions"
service_account: "github-actions@github-iam-387915.iam.gserviceaccount.com"
export_environment_variables: true
create_credentials_file: true
- name: Copy binaries to Google Cloud Storage
if: ${{ inputs.profile == 'release' && matrix.stage == 'release' && contains(fromJSON('["gateway", "client"]'), matrix.name.image_name) }}
run: |
gcloud storage cp \
${BINARY_DEST_PATH} \
gs://firezone-staging-artifacts/firezone-${{ matrix.name.image_name }}/${{ inputs.sha }}/${{ matrix.arch.shortname }}
gcloud storage cp \
${BINARY_DEST_PATH}.sha256sum.txt \
gs://firezone-staging-artifacts/firezone-${{ matrix.name.image_name }}/${{ inputs.sha }}/${{ matrix.arch.shortname }}.sha256sum.txt
- name: Set up QEMU
uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3.3.0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
with:
# We are overriding the default buildkit version being used by Buildx. We need buildkit >= 12.0 and currently BuildX
# supports v0.11.6 https://github.com/docker/buildx/blob/b8739d74417f86aa8fc9aafb830a8ba656bdef0e/Dockerfile#L9.
# We should for any updates on buildx and on the setup-buildx-action itself.
driver-opts: |
image=moby/buildkit:v0.15.1
- uses: ./.github/actions/gcp-docker-login
id: login
with:
project: firezone-staging
- name: Build Version Tags
run: |
set -xe
version="${{ matrix.name.version }}"
MAJOR_VERSION="${version%%.*}"
MAJOR_MINOR_VERSION="${version%.*}"
echo "MAJOR_VERSION=${MAJOR_VERSION}" >> $GITHUB_ENV
echo "MAJOR_MINOR_VERSION=${MAJOR_MINOR_VERSION}" >> $GITHUB_ENV
- name: Docker meta
id: meta
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
with:
images: ${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }}
# We only version client and gateway
tags: |
type=raw,value=latest
type=raw,value={{branch}}
type=raw,value=${{ inputs.sha }}
${{ matrix.name.version && format('type=raw,value={0}', matrix.name.version) }}
${{ matrix.name.version && format('type=raw,value={0}-{1}', matrix.name.version, inputs.sha) }}
${{ matrix.name.version && format('type=raw,value={0}', env.MAJOR_VERSION) }}
${{ matrix.name.version && format('type=raw,value={0}', env.MAJOR_MINOR_VERSION) }}
- name: Sanitize github.ref_name
run: |
# `ref_name` contains `/`, '_' or '=' which is not a valid docker image tag
REF="${{ github.ref_name }}"
CACHE_TAG="${REF//[\/_=]/-}"
echo "CACHE_TAG=$CACHE_TAG" >> "$GITHUB_ENV"
- name: Build Docker images
id: build
uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
with:
platforms: ${{ matrix.arch.platform }}
build-args: |
PACKAGE=${{ matrix.name.package }}
TARGET=${{ matrix.arch.target }}
context: rust
cache-from: |
type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.name.image_name }}:${{ env.CACHE_TAG }}
type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.name.image_name }}:main
cache-to: |
type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.name.image_name }}:${{ env.CACHE_TAG }},mode=max
target: ${{ matrix.stage }}
outputs: type=image,name=${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }},push-by-digest=true,name-canonical=true,push=true
- name: Export digest
run: |
mkdir -p /tmp/digests/${{ matrix.name.image_name }}
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${{ matrix.name.image_name }}/${digest#sha256:}"
- name: Upload digest artifact
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
with:
overwrite: true
name: ${{ matrix.image_prefix && format('{0}-', matrix.image_prefix) || '' }}${{ matrix.name.image_name }}-${{ inputs.sha }}-digest-${{ matrix.arch.shortname }}
path: /tmp/digests/${{ matrix.name.image_name }}
if-no-files-found: error
retention-days: 1
- name: Output image name
id: image-name
run: echo "${{ matrix.name.image_name }}_image=${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }}" >> $GITHUB_OUTPUT
merge-docker-artifacts:
name: merge-${{ matrix.image.name }}
needs: data-plane-linux
if: ${{ always() }}
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
# Copy input vars to matrix vars to conditionally exclude them
image_prefix:
- ${{ inputs.image_prefix }}
# Exclude http-test-server from perf image builds
exclude:
- { image_prefix: perf, image: { name: http-test-server } }
image:
- name: relay
- name: gateway
# mark:next-gateway-version
version: 1.4.5
- name: client
# mark:next-client-version
version: 1.0.6
- name: http-test-server
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.sha }}
- uses: ./.github/actions/gcp-docker-login
id: login
with:
project: firezone-staging
- name: Download digests
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
pattern: ${{ matrix.image_prefix && format('{0}-', matrix.image_prefix) || '' }}${{ matrix.image.name }}-${{ inputs.sha }}-digest-*
merge-multiple: true
path: /tmp/digests/${{ matrix.image.name }}
- name: Display structure of downloaded artifacts
run: ls -R /tmp/digests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
with:
# We are overriding the default buildkit version being used by Buildx. We need buildkit >= 12.0 and currently BuildX
# supports v0.11.6 https://github.com/docker/buildx/blob/b8739d74417f86aa8fc9aafb830a8ba656bdef0e/Dockerfile#L9.
# We should for any updates on buildx and on the setup-buildx-action itself.
driver-opts: |
image=moby/buildkit:v0.15.1
- name: Build Version Tags
run: |
set -xe
version="${{ matrix.image.version }}"
MAJOR_VERSION="${version%%.*}"
MAJOR_MINOR_VERSION="${version%.*}"
echo "MAJOR_VERSION=${MAJOR_VERSION}" >> $GITHUB_ENV
echo "MAJOR_MINOR_VERSION=${MAJOR_MINOR_VERSION}" >> $GITHUB_ENV
- name: Docker meta
id: meta
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
with:
images: ${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.image.name }}
tags: |
type=raw,value=latest
type=raw,value={{branch}}
type=raw,value=${{ inputs.sha }}
${{ matrix.image.version && format('type=raw,value={0}', matrix.image.version) }}
${{ matrix.image.version && format('type=raw,value={0}-{1}', matrix.image.version, inputs.sha) }}
${{ matrix.image.version && format('type=raw,value={0}', env.MAJOR_VERSION) }}
${{ matrix.image.version && format('type=raw,value={0}', env.MAJOR_MINOR_VERSION) }}
- name: Create manifest list and push
working-directory: /tmp/digests/${{ matrix.image.name }}
run: |
tags=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")
sources=$(printf '${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.image.name }}@sha256:%s ' *)
echo "$sources"
docker buildx imagetools create $tags $sources
docker buildx imagetools inspect "${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.image.name }}"