fix(ci): move rust cache to Azure; docker on main (#10169)

We are _very much_ over our GHA cache limit of 10 GB so in an effort to
keep evictions to a minimum, we update the Rust SCCACHE to only write on
`main` and the Docker elixir and data plane image build steps to do the
same.

Fixes #10145

.github/actions/setup-android/action.yml

@@ -1,12 +1,16 @@
 # From https://tauri.app/v1/guides/getting-started/prerequisites
 name: "Setup Android"
 description: "Sets up the dependencies for building our Android app"
+inputs:
+  sccache_azure_connection_string:
+    description: "Azure connection string for sccache"
 runs:
   using: "composite"
   steps:
     - uses: ./.github/actions/setup-rust
       with:
         targets: armv7-linux-androideabi aarch64-linux-android x86_64-linux-android i686-linux-android
+        sccache_azure_connection_string: ${{ inputs.sccache_azure_connection_string }}
 
     - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
       with:

.github/actions/setup-rust/action.yml

@@ -2,6 +2,8 @@
 name: "Setup Rust"
 description: "Sets up the correct Rust version and caching via sccache and a GCP backend"
 inputs:
+  sccache_azure_connection_string:
+    description: "Azure connection string for sccache"
   targets:
     description: "Additional targets to install"
     required: false
@@ -34,10 +36,7 @@ runs:
   using: "composite"
   steps:
     - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
-    - run: |
-        echo "RUSTC_WRAPPER=$SCCACHE_PATH" >> $GITHUB_ENV
-        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
-      shell: bash
+
     - name: Disable Windows Defender
       if: ${{ runner.os == 'Windows' }}
       run: Set-MpPreference -DisableRealtimeMonitoring $true
@@ -49,10 +48,24 @@ runs:
         echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> $GITHUB_ENV
       shell: bash
       working-directory: ./rust
+
     - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # v1
+      id: toolchain
       with:
         toolchain: ${{ env.RUST_TOOLCHAIN }}
         components: rustfmt,clippy
+
+    # We use Azure Blob Storage for sccache because credits and GHA cache is too small (10 GB).
+    # For this to work, you need an Azure Storage account and a blob container named `sccache`.
+    # The connection string here can be found under Storage Account -> Settings -> Security + networking -> Access keys.
+    - name: Configure sccache
+      shell: bash
+      run: |
+        echo "SCCACHE_GHA_ENABLED=false" >> $GITHUB_ENV
+        echo "SCCACHE_AZURE_CONNECTION_STRING=${{ inputs.sccache_azure_connection_string }}" >> $GITHUB_ENV
+        echo "SCCACHE_AZURE_BLOB_CONTAINER=sccache" >> $GITHUB_ENV
+        echo "RUSTC_WRAPPER=$SCCACHE_PATH" >> $GITHUB_ENV
+
     - if: inputs.targets != ''
       run: rustup target add ${{ inputs.targets }}
       shell: bash
@@ -61,10 +74,8 @@ runs:
       id: nightly
       run: |
         NIGHTLY="nightly-2025-05-30"
-
         rustup toolchain install $NIGHTLY
         rustup component add rust-src --toolchain $NIGHTLY
-
         echo "nightly=$NIGHTLY" >> $GITHUB_OUTPUT
       shell: bash
 

.github/workflows/_build_artifacts.yml

@@ -94,8 +94,26 @@ jobs:
           REF="${{ github.ref_name }}"
           CACHE_TAG="${REF//[\/_=]/-}"
           echo "CACHE_TAG=$CACHE_TAG" >> "$GITHUB_ENV"
-      - name: Build and push control plane images
-        id: build
+      # PRs & non-main branches: read-only cache
+      - name: Build and push control plane images (read-only cache)
+        if: ${{ github.ref != 'refs/heads/main' }}
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          build-args: ${{ matrix.build-args }}
+          target: ${{ matrix.target }}
+          context: elixir
+          cache-from: |
+            type=gha,scope=${{ matrix.image_name }}:${{ env.CACHE_TAG }}
+            type=gha,scope=${{ matrix.image_name }}:main
+          # no cache-to here -> read-only
+          push: true
+          tags: |
+            ${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ inputs.sha }}
+            ${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
+
+      # main: read/write cache
+      - name: Build and push control plane images (read/write cache)
+        if: ${{ github.ref == 'refs/heads/main' }}
         uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
         with:
           build-args: ${{ matrix.build-args }}
@@ -140,6 +158,7 @@ jobs:
       - uses: ./.github/actions/setup-rust
         with:
           targets: ${{ matrix.target }}
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - name: Build binaries
         shell: bash
         run: |
@@ -272,6 +291,7 @@ jobs:
       - uses: ./.github/actions/setup-rust
         with:
           targets: ${{ matrix.arch.target }}
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - name: Cache toolchain
         uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # 4.2.3
         with:
@@ -340,8 +360,27 @@ jobs:
           REF="${{ github.ref_name }}"
           CACHE_TAG="${REF//[\/_=]/-}"
           echo "CACHE_TAG=$CACHE_TAG" >> "$GITHUB_ENV"
-      - name: Build Docker images
-        id: build
+      # PRs & non-main branches: read-only cache
+      - name: Build Docker images (read-only cache)
+        if: ${{ github.ref != 'refs/heads/main' }}
+        id: build_ro
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          platforms: ${{ matrix.arch.platform }}
+          build-args: |
+            PACKAGE=${{ matrix.name.package }}
+            TARGET=${{ matrix.arch.target }}
+          context: rust
+          cache-from: |
+            type=gha,scope=${{ matrix.name.image_name }}:${{ env.CACHE_TAG }}
+            type=gha,scope=${{ matrix.name.image_name }}:main
+          # no cache-to -> read-only
+          target: ${{ matrix.stage }}
+          outputs: type=image,name=${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }},push-by-digest=true,name-canonical=true,push=true
+      # main: read/write cache
+      - name: Build Docker images (read/write cache)
+        if: ${{ github.ref == 'refs/heads/main' }}
+        id: build_rw
         uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
         with:
           platforms: ${{ matrix.arch.platform }}
@@ -359,7 +398,7 @@ jobs:
       - name: Export digest
         run: |
           mkdir -p /tmp/digests/${{ matrix.name.image_name }}
-          digest="${{ steps.build.outputs.digest }}"
+          digest="${{ github.ref == 'refs/heads/main' && steps.build_rw.outputs.digest || steps.build_ro.outputs.digest }}"
           touch "/tmp/digests/${{ matrix.name.image_name }}/${digest#sha256:}"
       - name: Upload digest artifact
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

.github/workflows/_kotlin.yml

@@ -18,6 +18,8 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-android
+        with:
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - name: Run linter
         run: ./gradlew spotlessCheck
 
@@ -66,6 +68,8 @@ jobs:
         with:
           fetch-tags: true # Otherwise we cannot embed the correct version into the build.
       - uses: ./.github/actions/setup-android
+        with:
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - name: Build the release package
         env:
           KEYSTORE_BASE64: ${{ secrets.GOOGLE_UPLOAD_KEYSTORE_BASE64 }}
@@ -122,6 +126,8 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-android
+        with:
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - name: Build debug APK
         run: |
           ./gradlew assembleDebug

.github/workflows/_rust.yml

@@ -29,6 +29,8 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-rust
         id: setup-rust
+        with:
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - uses: ./.github/actions/setup-tauri-v2
         timeout-minutes: 10
       - uses: taiki-e/install-action@d31232495ad76f47aad66e3501e47780b49f0f3e # v2.57.5
@@ -76,6 +78,8 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-rust
         id: setup-rust
+        with:
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - uses: ./.github/actions/setup-tauri-v2
       - uses: taiki-e/install-action@d31232495ad76f47aad66e3501e47780b49f0f3e # v2.57.5
         env:
@@ -136,6 +140,8 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-rust
         id: setup-rust
+        with:
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - uses: taiki-e/install-action@d31232495ad76f47aad66e3501e47780b49f0f3e # v2.57.5
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -161,6 +167,8 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-rust
+        with:
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - uses: ./.github/actions/setup-tauri-v2
         timeout-minutes: 10
       - run: scripts/tests/${{ matrix.test }}

.github/workflows/_swift.yml

@@ -67,6 +67,7 @@ jobs:
       - uses: ./.github/actions/setup-rust
         with:
           targets: ${{ matrix.rust-targets }}
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         name: Restore Swift DerivedData Cache
         id: cache

.github/workflows/_tauri.yml

@@ -72,6 +72,8 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-node
       - uses: ./.github/actions/setup-rust
+        with:
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - uses: ./.github/actions/setup-tauri-v2
         timeout-minutes: 10
         with:
@@ -138,6 +140,8 @@ jobs:
           fetch-tags: true # Otherwise we cannot embed the correct version into the build.
       - uses: ./.github/actions/setup-node
       - uses: ./.github/actions/setup-rust
+        with:
+          sccache_azure_connection_string: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
       - uses: ./.github/actions/setup-tauri-v2
         # Installing new packages can take time
         timeout-minutes: 10