ci: use sccache for Rust (#2402)

Our caches in GitHub actions are hopelessly overflowing, plus for the
Kotlin and Swift jobs, we don't seem to be doing a particularly good job
at caching the build outputs because those jobs take forever.

Instead of using GitHub actions, this PR configures `sccache` for all
Rust compilation commands and uses a GCP bucket to store the artifacts.
This speeds up some of the builds a fair bit. Android now finishes in
~6minutes.

Apart from the self-hosted MacOS 14 runner, the Swift jobs are slow but
still a lot faster than what we currently have.

Windows seems to be quite slow at compiling / fetching artefacts which
is negatively impacted by this change because they now have to be
fetched from the bucket.

Overall, I think this is a net-positive though and should be much easier
to maintain going forward.

---------

Co-authored-by: Jamil <jamilbk@users.noreply.github.com>

.github/actions/setup-rust/action.yml

@@ -0,0 +1,38 @@
+name: "Setup Rust"
+description: "Sets up the correct Rust version and caching via sccache and a GCP backend"
+inputs:
+  targets:
+    description: "Additional targets to install"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - id: auth
+      uses: google-github-actions/auth@v1
+      with:
+        workload_identity_provider: "projects/397012414171/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions"
+        service_account: "github-actions@github-iam-387915.iam.gserviceaccount.com"
+    - run: |
+        echo "SCCACHE_GCS_BUCKET=firezone-sccache" >> $GITHUB_ENV
+        echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> $GITHUB_ENV
+      shell: bash
+    - uses: mozilla-actions/sccache-action@v0.0.3
+    - run: echo "RUSTC_WRAPPER=$SCCACHE_PATH" >> $GITHUB_ENV
+      shell: bash
+    - name: Extract Rust version
+      run: |
+        RUST_TOOLCHAIN=$(grep 'channel' rust-toolchain.toml | awk -F '"' '{print $2}')
+        echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> $GITHUB_ENV
+      shell: bash
+      working-directory: ./rust
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ env.RUST_TOOLCHAIN }}
+        components: rustfmt,clippy
+    - run: rustup target add ${{ inputs.targets }}
+      if: inputs.targets != ''
+      shell: bash
+    - name: Start sccache
+      run: $SCCACHE_PATH --start-server
+      shell: bash

.github/workflows/kotlin.yml

@@ -2,6 +2,10 @@ name: Kotlin
 on:
   workflow_call:
 
+permissions:
+  contents: 'read'
+  id-token: 'write'
+
 jobs:
   static-analysis:
     runs-on: ubuntu-22.04
@@ -32,12 +36,9 @@ jobs:
         working-directory: ./kotlin/android
     steps:
       - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
+      - uses: ./.github/actions/setup-rust
         with:
-          workspaces: ./rust
-          key: ubuntu-22.04-${{ runner.arch }}
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - uses: dtolnay/rust-toolchain@stable
+          targets: armv7-linux-androideabi aarch64-linux-android x86_64-linux-android
       - uses: actions/setup-java@v3
         with:
           distribution: oracle

.github/workflows/rust.yml

@@ -6,6 +6,10 @@ defaults:
   run:
     working-directory: ./rust
 
+permissions:
+  contents: 'read'
+  id-token: 'write'
+
 jobs:
   static-analysis:
     strategy:
@@ -27,14 +31,7 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     steps:
       - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: ./rust
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-          key: ${{ matrix.runs-on }}-${{ runner.arch }}
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          components: rustfmt,clippy
+      - uses: ./.github/actions/setup-rust
       - run: cargo fmt -- --check
       - run: |
           cargo doc --all-features --no-deps --document-private-items ${{ matrix.packages }}
@@ -42,6 +39,7 @@ jobs:
           RUSTDOCFLAGS: "-D warnings"
       - run: |
           cargo clippy --all-targets --all-features ${{ matrix.packages }} -- -D warnings
+
   test:
     strategy:
       fail-fast: false
@@ -63,13 +61,7 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     steps:
       - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          # Prevents runner.os caches from conflicting
-          key: ${{ matrix.runs-on }}-${{ runner.arch }}
-          workspaces: ./rust
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - uses: dtolnay/rust-toolchain@stable
+      - uses: ./.github/actions/setup-rust
       - run: cargo test --all-features ${{ matrix.packages }}
 
   smoke-test-relay:
@@ -79,10 +71,5 @@ jobs:
         working-directory: ./rust/relay
     steps:
       - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: ./rust
-          key: ubuntu-22.04-${{ runner.arch }}
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - uses: dtolnay/rust-toolchain@stable
+      - uses: ./.github/actions/setup-rust
       - run: ./run_smoke_test.sh

.github/workflows/static-analysis.yml

@@ -7,12 +7,6 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: ./rust
-          key: ubuntu-22.04-${{ runner.arch }}
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - uses: dtolnay/rust-toolchain@stable
       - name: Check version is up to date
         run: |
           make version

.github/workflows/swift.yml

@@ -39,17 +39,15 @@ jobs:
             destination: generic/platform=iOS
     permissions:
       contents: read
+      id-token: 'write'
     defaults:
       run:
         working-directory: ./swift/apple
     steps:
       - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
+      - uses: ./.github/actions/setup-rust
         with:
-          workspaces: ./rust
-          key: ${{ matrix.runs-on }}-${{ runner.arch }}
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - uses: dtolnay/rust-toolchain@stable
+          targets: aarch64-apple-darwin aarch64-apple-ios
       - uses: actions/cache/restore@v3
         name: Restore Swift DerivedData Cache
         with:

rust/connlib/clients/android/connlib/build.gradle.kts

@@ -99,6 +99,8 @@ cargo {
     module = "../"
     libname = "connlib"
     targets = listOf("arm", "arm64", "x86_64")
+    targetDirectory = "../../../../target"
+    targetIncludes = arrayOf("connlib.dylib")
 }
 
 tasks.register("copyJniSharedObjectsDebug") {

rust/connlib/clients/apple/build-rust.sh

@@ -28,6 +28,9 @@ for var in $(env | awk -F= '{print $1}'); do
   && [[ "$var" != "SYMROOT" ]] \
   && [[ "$var" != "SRCROOT" ]] \
   && [[ "$var" != "TARGETED_DEVICE_FAMILY" ]] \
+  && [[ "$var" != "RUSTC_WRAPPER" ]] \
+  && [[ "$var" != "SCCACHE_GCS_BUCKET" ]] \
+  && [[ "$var" != "SCCACHE_GCS_RW_MODE" ]] \
   && [[ "$var" != "CONNLIB_TARGET_DIR" ]]; then
   unset $var
   fi

rust/rust-toolchain.toml

@@ -1,12 +1,2 @@
 [toolchain]
 channel = "1.73.0"
-components = ["rustfmt", "clippy"]
-targets = [
-  "x86_64-unknown-linux-musl",
-  "x86_64-linux-android",
-  "x86_64-apple-darwin",
-  "aarch64-apple-darwin",
-  "aarch64-apple-ios",
-  "aarch64-linux-android",
-  "armv7-linux-androideabi",
-]