From 3c55ddcd1e0198221d90209abe4e49e9987b2ac5 Mon Sep 17 00:00:00 2001 From: Jamil Date: Sat, 9 Aug 2025 21:29:12 -0400 Subject: [PATCH] fix(ci): move rust cache to Azure; docker on `main` (#10169) We are _very much_ over our GHA cache limit of 10 GB so in an effort to keep evictions to a minimum, we update the Rust SCCACHE to only write on `main` and the Docker elixir and data plane image build steps to do the same. Fixes #10145 --- .github/actions/setup-android/action.yml | 4 ++ .github/actions/setup-rust/action.yml | 23 ++++++++--- .github/workflows/_build_artifacts.yml | 49 +++++++++++++++++++++--- .github/workflows/_kotlin.yml | 6 +++ .github/workflows/_rust.yml | 8 ++++ .github/workflows/_swift.yml | 1 + .github/workflows/_tauri.yml | 4 ++ 7 files changed, 84 insertions(+), 11 deletions(-) diff --git a/.github/actions/setup-android/action.yml b/.github/actions/setup-android/action.yml index 779fd46bc..83a6b1302 100644 --- a/.github/actions/setup-android/action.yml +++ b/.github/actions/setup-android/action.yml @@ -1,12 +1,16 @@ # From https://tauri.app/v1/guides/getting-started/prerequisites name: "Setup Android" description: "Sets up the dependencies for building our Android app" +inputs: + sccache_azure_connection_string: + description: "Azure connection string for sccache" runs: using: "composite" steps: - uses: ./.github/actions/setup-rust with: targets: armv7-linux-androideabi aarch64-linux-android x86_64-linux-android i686-linux-android + sccache_azure_connection_string: ${{ inputs.sccache_azure_connection_string }} - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 with: diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index ed1fb04b7..570dc12f7 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -2,6 +2,8 @@ name: "Setup Rust" description: "Sets up the correct Rust version and caching via sccache and a GCP backend" inputs: + sccache_azure_connection_string: + description: "Azure connection string for sccache" targets: description: "Additional targets to install" required: false @@ -34,10 +36,7 @@ runs: using: "composite" steps: - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - - run: | - echo "RUSTC_WRAPPER=$SCCACHE_PATH" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - shell: bash + - name: Disable Windows Defender if: ${{ runner.os == 'Windows' }} run: Set-MpPreference -DisableRealtimeMonitoring $true @@ -49,10 +48,24 @@ runs: echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> $GITHUB_ENV shell: bash working-directory: ./rust + - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # v1 + id: toolchain with: toolchain: ${{ env.RUST_TOOLCHAIN }} components: rustfmt,clippy + + # We use Azure Blob Storage for sccache because credits and GHA cache is too small (10 GB). + # For this to work, you need an Azure Storage account and a blob container named `sccache`. + # The connection string here can be found under Storage Account -> Settings -> Security + networking -> Access keys. + - name: Configure sccache + shell: bash + run: | + echo "SCCACHE_GHA_ENABLED=false" >> $GITHUB_ENV + echo "SCCACHE_AZURE_CONNECTION_STRING=${{ inputs.sccache_azure_connection_string }}" >> $GITHUB_ENV + echo "SCCACHE_AZURE_BLOB_CONTAINER=sccache" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=$SCCACHE_PATH" >> $GITHUB_ENV + - if: inputs.targets != '' run: rustup target add ${{ inputs.targets }} shell: bash @@ -61,10 +74,8 @@ runs: id: nightly run: | NIGHTLY="nightly-2025-05-30" - rustup toolchain install $NIGHTLY rustup component add rust-src --toolchain $NIGHTLY - echo "nightly=$NIGHTLY" >> $GITHUB_OUTPUT shell: bash diff --git a/.github/workflows/_build_artifacts.yml b/.github/workflows/_build_artifacts.yml index 3a0535b2a..3a6c5b62e 100644 --- a/.github/workflows/_build_artifacts.yml +++ b/.github/workflows/_build_artifacts.yml @@ -94,8 +94,26 @@ jobs: REF="${{ github.ref_name }}" CACHE_TAG="${REF//[\/_=]/-}" echo "CACHE_TAG=$CACHE_TAG" >> "$GITHUB_ENV" - - name: Build and push control plane images - id: build + # PRs & non-main branches: read-only cache + - name: Build and push control plane images (read-only cache) + if: ${{ github.ref != 'refs/heads/main' }} + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + build-args: ${{ matrix.build-args }} + target: ${{ matrix.target }} + context: elixir + cache-from: | + type=gha,scope=${{ matrix.image_name }}:${{ env.CACHE_TAG }} + type=gha,scope=${{ matrix.image_name }}:main + # no cache-to here -> read-only + push: true + tags: | + ${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ inputs.sha }} + ${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ env.CACHE_TAG }} + + # main: read/write cache + - name: Build and push control plane images (read/write cache) + if: ${{ github.ref == 'refs/heads/main' }} uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: build-args: ${{ matrix.build-args }} @@ -140,6 +158,7 @@ jobs: - uses: ./.github/actions/setup-rust with: targets: ${{ matrix.target }} + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - name: Build binaries shell: bash run: | @@ -272,6 +291,7 @@ jobs: - uses: ./.github/actions/setup-rust with: targets: ${{ matrix.arch.target }} + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - name: Cache toolchain uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # 4.2.3 with: @@ -340,8 +360,27 @@ jobs: REF="${{ github.ref_name }}" CACHE_TAG="${REF//[\/_=]/-}" echo "CACHE_TAG=$CACHE_TAG" >> "$GITHUB_ENV" - - name: Build Docker images - id: build + # PRs & non-main branches: read-only cache + - name: Build Docker images (read-only cache) + if: ${{ github.ref != 'refs/heads/main' }} + id: build_ro + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + platforms: ${{ matrix.arch.platform }} + build-args: | + PACKAGE=${{ matrix.name.package }} + TARGET=${{ matrix.arch.target }} + context: rust + cache-from: | + type=gha,scope=${{ matrix.name.image_name }}:${{ env.CACHE_TAG }} + type=gha,scope=${{ matrix.name.image_name }}:main + # no cache-to -> read-only + target: ${{ matrix.stage }} + outputs: type=image,name=${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }},push-by-digest=true,name-canonical=true,push=true + # main: read/write cache + - name: Build Docker images (read/write cache) + if: ${{ github.ref == 'refs/heads/main' }} + id: build_rw uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: platforms: ${{ matrix.arch.platform }} @@ -359,7 +398,7 @@ jobs: - name: Export digest run: | mkdir -p /tmp/digests/${{ matrix.name.image_name }} - digest="${{ steps.build.outputs.digest }}" + digest="${{ github.ref == 'refs/heads/main' && steps.build_rw.outputs.digest || steps.build_ro.outputs.digest }}" touch "/tmp/digests/${{ matrix.name.image_name }}/${digest#sha256:}" - name: Upload digest artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/.github/workflows/_kotlin.yml b/.github/workflows/_kotlin.yml index e2ef3639d..7c43ef6d5 100644 --- a/.github/workflows/_kotlin.yml +++ b/.github/workflows/_kotlin.yml @@ -18,6 +18,8 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/setup-android + with: + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - name: Run linter run: ./gradlew spotlessCheck @@ -66,6 +68,8 @@ jobs: with: fetch-tags: true # Otherwise we cannot embed the correct version into the build. - uses: ./.github/actions/setup-android + with: + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - name: Build the release package env: KEYSTORE_BASE64: ${{ secrets.GOOGLE_UPLOAD_KEYSTORE_BASE64 }} @@ -122,6 +126,8 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/setup-android + with: + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - name: Build debug APK run: | ./gradlew assembleDebug diff --git a/.github/workflows/_rust.yml b/.github/workflows/_rust.yml index 810464135..51ae6f784 100644 --- a/.github/workflows/_rust.yml +++ b/.github/workflows/_rust.yml @@ -29,6 +29,8 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/setup-rust id: setup-rust + with: + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - uses: ./.github/actions/setup-tauri-v2 timeout-minutes: 10 - uses: taiki-e/install-action@d31232495ad76f47aad66e3501e47780b49f0f3e # v2.57.5 @@ -76,6 +78,8 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/setup-rust id: setup-rust + with: + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - uses: ./.github/actions/setup-tauri-v2 - uses: taiki-e/install-action@d31232495ad76f47aad66e3501e47780b49f0f3e # v2.57.5 env: @@ -136,6 +140,8 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/setup-rust id: setup-rust + with: + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - uses: taiki-e/install-action@d31232495ad76f47aad66e3501e47780b49f0f3e # v2.57.5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -161,6 +167,8 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/setup-rust + with: + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - uses: ./.github/actions/setup-tauri-v2 timeout-minutes: 10 - run: scripts/tests/${{ matrix.test }} diff --git a/.github/workflows/_swift.yml b/.github/workflows/_swift.yml index fb92b734a..64eaffd7f 100644 --- a/.github/workflows/_swift.yml +++ b/.github/workflows/_swift.yml @@ -67,6 +67,7 @@ jobs: - uses: ./.github/actions/setup-rust with: targets: ${{ matrix.rust-targets }} + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 name: Restore Swift DerivedData Cache id: cache diff --git a/.github/workflows/_tauri.yml b/.github/workflows/_tauri.yml index f0894edf4..480f3c97f 100644 --- a/.github/workflows/_tauri.yml +++ b/.github/workflows/_tauri.yml @@ -72,6 +72,8 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/setup-node - uses: ./.github/actions/setup-rust + with: + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - uses: ./.github/actions/setup-tauri-v2 timeout-minutes: 10 with: @@ -138,6 +140,8 @@ jobs: fetch-tags: true # Otherwise we cannot embed the correct version into the build. - uses: ./.github/actions/setup-node - uses: ./.github/actions/setup-rust + with: + sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - uses: ./.github/actions/setup-tauri-v2 # Installing new packages can take time timeout-minutes: 10