chore(infra): ditch gcp registry for ghcr.io (#9913)

Google Cloud Artifact registry and Cloud storage is a significant cost. GitHub, on the other hand, is completely free due to our being a public repository. Hence, it makes sense to ditch GCP for GHCR. To do this, we move all "staging" artifacts to GHCR. These will then be used in the infra repo to push to GCP for deploys - we probably still want pulls for our infra to hit GCP and not GitHub. One big element of this is that we potentially lose sccache, so I'll be checking the compile time of this PR and looking for alternatives that don't involve such a massive cloud bill.
2026-01-27 10:18:54 +00:00 · 2025-07-19 10:00:30 -04:00
parent 318ce24403
commit a8f93d24a3
9 changed files with 69 additions and 171 deletions
--- a/.github/actions/gcp-docker-login/action.yml
+++ b/.github/actions/gcp-docker-login/action.yml
@@ -1,68 +0,0 @@
-name: "GCP docker registry login"
-description: "Login to the GCP docker registry"
-outputs:
-  registry:
-    description: "The full name of the registry we logged into"
-    value: ${{ format('us-east1-docker.pkg.dev/firezone-staging') }}
-runs:
-  using: "composite"
-  steps:
-    - id: auth1
-      uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
-      continue-on-error: true
-      with:
-        token_format: access_token
-        workload_identity_provider: "projects/397012414171/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions"
-        service_account: "github-actions@github-iam-387915.iam.gserviceaccount.com"
-        export_environment_variables: true
-        create_credentials_file: true
-
-    - id: auth2
-      if: ${{ steps.auth1.outcome == 'failure' }}
-      uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
-      continue-on-error: true
-      with:
-        token_format: access_token
-        workload_identity_provider: "projects/397012414171/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions"
-        service_account: "github-actions@github-iam-387915.iam.gserviceaccount.com"
-        export_environment_variables: true
-        create_credentials_file: true
-
-    - id: auth3
-      if: ${{ steps.auth2.outcome == 'failure' }}
-      uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
-      with:
-        token_format: access_token
-        workload_identity_provider: "projects/397012414171/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions"
-        service_account: "github-actions@github-iam-387915.iam.gserviceaccount.com"
-        export_environment_variables: true
-        create_credentials_file: true
-
-    - name: Login to Google Artifact Registry
-      if: ${{ runner.os == 'Linux' }}
-      uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
-      with:
-        registry: "us-east1-docker.pkg.dev"
-        username: oauth2accesstoken
-        password: ${{ steps.auth3.outputs.access_token || steps.auth2.outputs.access_token || steps.auth1.outputs.access_token }}
-
-    # DockerHub has stupid rate limits (see https://www.docker.com/increase-rate-limits/)
-    # Use Google's public mirror instead: https://cloud.google.com/artifact-registry/docs/pull-cached-dockerhub-images
-    - name: Setup registry mirror
-      if: ${{ runner.os == 'Linux' }}
-      shell: bash
-      run: |
-        # Create daemon.json if it doesn't exist
-        if [ ! -f /etc/docker/daemon.json ]; then
-          echo '{}' | sudo tee /etc/docker/daemon.json > /dev/null
-        fi
-
-        # Add or update registry-mirrors configuration
-        echo "$(jq '. += { "registry-mirrors": ["https://mirror.gcr.io"] }' /etc/docker/daemon.json)" > new_daemon.json
-        sudo mv new_daemon.json /etc/docker/daemon.json
-        sudo service docker restart
-
-    - name: Print docker system info
-      if: ${{ runner.os == 'Linux' }}
-      shell: bash
-      run: docker system info
--- a/.github/actions/ghcr-docker-login/action.yml
+++ b/.github/actions/ghcr-docker-login/action.yml
@@ -0,0 +1,22 @@
+name: "GHCR docker registry login"
+description: "Login to the GitHub container registry"
+
+inputs:
+  github_token:
+    description: "GitHub token to use for authentication"
+    required: true
+
+outputs:
+  registry:
+    description: "The full name of the registry we logged into"
+    value: ${{ format('ghcr.io') }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ inputs.github_token }}
--- a/.github/actions/setup-rust/action.yml
+++ b/.github/actions/setup-rust/action.yml
@@ -33,15 +33,6 @@ outputs:
 runs:
  using: "composite"
  steps:
-    - uses: ./.github/actions/gcp-docker-login
-    - run: |
-        echo "SCCACHE_GCS_BUCKET=firezone-staging-sccache" >> $GITHUB_ENV
-        echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> $GITHUB_ENV
-      shell: bash
-    - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
-    - run: echo "RUSTC_WRAPPER=$SCCACHE_PATH" >> $GITHUB_ENV
-      shell: bash
-
    - name: Disable Windows Defender
      if: ${{ runner.os == 'Windows' }}
      run: Set-MpPreference -DisableRealtimeMonitoring $true
@@ -71,9 +62,3 @@ runs:

        echo "nightly=$NIGHTLY" >> $GITHUB_OUTPUT
      shell: bash
-
-    - name: Start sccache
-      run: $SCCACHE_PATH --start-server
-      shell: bash
-      env:
-        SCCACHE_CONF: ".github/actions/setup-rust/sccache.toml"
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -18,7 +18,7 @@ updates:
      - "/"
      # Dependabot doesn't look in these by default
      - "/.github/actions/create-sentry-release"
-      - "/.github/actions/gcp-docker-login"
+      - "/.github/actions/ghcr-docker-login"
      - "/.github/actions/setup-android"
      - "/.github/actions/setup-elixir"
      - "/.github/actions/setup-node"
--- a/.github/workflows/_build_artifacts.yml
+++ b/.github/workflows/_build_artifacts.yml
@@ -41,6 +41,7 @@ permissions:
  # write permission is required to create a github release
  contents: write
  id-token: write
+  packages: write

 jobs:
  control-plane:
@@ -76,8 +77,10 @@ jobs:
          ref: ${{ inputs.sha }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
-      - uses: ./.github/actions/gcp-docker-login
+      - uses: ./.github/actions/ghcr-docker-login
        id: login
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
@@ -99,10 +102,10 @@ jobs:
          target: ${{ matrix.target }}
          context: elixir
          cache-from: |
-            type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
-            type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.image_name }}:main
+            type=registry,ref=${{ steps.login.outputs.registry }}/firezone/cache/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
+            type=registry,ref=${{ steps.login.outputs.registry }}/firezone/cache/${{ matrix.image_name }}:main
          cache-to: |
-            type=registry,ref=${{steps.login.outputs.registry}}/cache/${{ matrix.image_name}}:${{ env.CACHE_TAG }},mode=max
+            type=registry,ref=${{steps.login.outputs.registry}}/firezone/cache/${{ matrix.image_name}}:${{ env.CACHE_TAG }},mode=max
          push: true
          tags: |
            ${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_name }}:${{ inputs.sha }}
@@ -263,8 +266,10 @@ jobs:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ inputs.sha }}
-      - uses: ./.github/actions/gcp-docker-login
+      - uses: ./.github/actions/ghcr-docker-login
        id: login
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
      - uses: ./.github/actions/setup-rust
        with:
          targets: ${{ matrix.arch.target }}
@@ -370,10 +375,10 @@ jobs:
            TARGET=${{ matrix.arch.target }}
          context: rust
          cache-from: |
-            type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.name.image_name }}:${{ env.CACHE_TAG }}
-            type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.name.image_name }}:main
+            type=registry,ref=${{ steps.login.outputs.registry }}/firezone/cache/${{ matrix.name.image_name }}:${{ env.CACHE_TAG }}
+            type=registry,ref=${{ steps.login.outputs.registry }}/firezone/cache/${{ matrix.name.image_name }}:main
          cache-to: |
-            type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.name.image_name }}:${{ env.CACHE_TAG }},mode=max
+            type=registry,ref=${{ steps.login.outputs.registry }}/firezone/cache/${{ matrix.name.image_name }}:${{ env.CACHE_TAG }},mode=max
          target: ${{ matrix.stage }}
          outputs: type=image,name=${{ steps.login.outputs.registry }}/firezone/${{ matrix.image_prefix && format('{0}/', matrix.image_prefix) || '' }}${{ matrix.name.image_name }},push-by-digest=true,name-canonical=true,push=true
      - name: Export digest
@@ -422,8 +427,10 @@ jobs:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ inputs.sha }}
-      - uses: ./.github/actions/gcp-docker-login
+      - uses: ./.github/actions/ghcr-docker-login
        id: login
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
      - name: Download digests
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
--- a/.github/workflows/_integration_tests.yml
+++ b/.github/workflows/_integration_tests.yml
@@ -6,7 +6,7 @@ on:
      domain_image:
        required: false
        type: string
-        default: "us-east1-docker.pkg.dev/firezone-staging/firezone/domain"
+        default: "ghcr.io/firezone/domain"
      domain_tag:
        required: false
        type: string
@@ -14,7 +14,7 @@ on:
      api_image:
        required: false
        type: string
-        default: "us-east1-docker.pkg.dev/firezone-staging/firezone/api"
+        default: "ghcr.io/firezone/api"
      api_tag:
        required: false
        type: string
@@ -22,7 +22,7 @@ on:
      web_image:
        required: false
        type: string
-        default: "us-east1-docker.pkg.dev/firezone-staging/firezone/web"
+        default: "ghcr.io/firezone/web"
      web_tag:
        required: false
        type: string
@@ -30,7 +30,7 @@ on:
      elixir_image:
        required: false
        type: string
-        default: "us-east1-docker.pkg.dev/firezone-staging/firezone/elixir"
+        default: "ghcr.io/firezone/elixir"
      elixir_tag:
        required: false
        type: string
@@ -38,7 +38,7 @@ on:
      relay_image:
        required: false
        type: string
-        default: "us-east1-docker.pkg.dev/firezone-staging/firezone/debug/relay"
+        default: "ghcr.io/firezone/debug/relay"
      relay_tag:
        required: false
        type: string
@@ -46,7 +46,7 @@ on:
      gateway_image:
        required: false
        type: string
-        default: "us-east1-docker.pkg.dev/firezone-staging/firezone/debug/gateway"
+        default: "ghcr.io/firezone/debug/gateway"
      gateway_tag:
        required: false
        type: string
@@ -54,7 +54,7 @@ on:
      client_image:
        required: false
        type: string
-        default: "us-east1-docker.pkg.dev/firezone-staging/firezone/debug/client"
+        default: "ghcr.io/firezone/debug/client"
      client_tag:
        required: false
        type: string
@@ -62,7 +62,7 @@ on:
      http_test_server_image:
        required: false
        type: string
-        default: "us-east1-docker.pkg.dev/firezone-staging/firezone/debug/http-test-server"
+        default: "ghcr.io/firezone/debug/http-test-server"
      http_test_server_tag:
        required: false
        type: string
@@ -115,7 +115,9 @@ jobs:
          - name: systemd/dns-systemd-resolved
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: ./.github/actions/gcp-docker-login
+      - uses: ./.github/actions/ghcr-docker-login
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
      - name: Seed database
        run: docker compose run elixir /bin/sh -c 'cd apps/domain && mix ecto.migrate --migrations-path priv/repo/migrations --migrations-path priv/repo/manual_migrations && mix ecto.seed'
      - name: Start docker compose in the background
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -281,17 +281,17 @@ jobs:
      id-token: write
      pull-requests: write
    env:
-      API_IMAGE: "us-east1-docker.pkg.dev/firezone-staging/firezone/api"
+      API_IMAGE: "ghcr.io/firezone/api"
      API_TAG: ${{ github.sha }}
-      WEB_IMAGE: "us-east1-docker.pkg.dev/firezone-staging/firezone/web"
+      WEB_IMAGE: "ghcr.io/firezone/web"
      WEB_TAG: ${{ github.sha }}
-      ELIXIR_IMAGE: "us-east1-docker.pkg.dev/firezone-staging/firezone/elixir"
+      ELIXIR_IMAGE: "ghcr.io/firezone/elixir"
      ELIXIR_TAG: ${{ github.sha }}
-      GATEWAY_IMAGE: "us-east1-docker.pkg.dev/firezone-staging/firezone/perf/gateway"
+      GATEWAY_IMAGE: "ghcr.io/firezone/perf/gateway"
      GATEWAY_TAG: ${{ github.sha }}
-      CLIENT_IMAGE: "us-east1-docker.pkg.dev/firezone-staging/firezone/perf/client"
+      CLIENT_IMAGE: "ghcr.io/firezone/perf/client"
      CLIENT_TAG: ${{ github.sha }}
-      RELAY_IMAGE: "us-east1-docker.pkg.dev/firezone-staging/firezone/perf/relay"
+      RELAY_IMAGE: "ghcr.io/firezone/perf/relay"
      RELAY_TAG: ${{ github.sha }}
    strategy:
      fail-fast: false
@@ -307,7 +307,9 @@ jobs:
          - relayed-udp-server2client
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: ./.github/actions/gcp-docker-login
+      - uses: ./.github/actions/ghcr-docker-login
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
      - name: Seed database
        run: docker compose run elixir /bin/sh -c 'cd apps/domain && mix ecto.seed --migrations-path priv/repo/migrations --migrations-path priv/repo/manual_migrations'
      - name: Start docker compose in the background
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -25,12 +25,8 @@ jobs:
    permissions:
      # Needed to upload artifacts to a release
      packages: write
-      # Needed to login to GCP
-      id-token: write
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: ./.github/actions/gcp-docker-login
-        id: login
      - name: Login to GitHub Container Registry
        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
        with:
@@ -73,45 +69,15 @@ jobs:
        run: |
          set -xe

-          SOURCE_TAG=${{ steps.login.outputs.registry }}/firezone/${{ steps.set-variables.outputs.artifact }}:${{ steps.set-variables.outputs.sha }}
+          SOURCE_TAG=ghcr.io/firezone/${{ steps.set-variables.outputs.artifact }}:${{ steps.set-variables.outputs.sha }}

          docker buildx imagetools create \
-            -t ghcr.io/firezone/${{ steps.set-variables.outputs.artifact }}:${{ steps.set-variables.outputs.sha }} \
            -t ghcr.io/firezone/${{ steps.set-variables.outputs.artifact }}:${{ steps.set-variables.outputs.version }} \
            -t ghcr.io/firezone/${{ steps.set-variables.outputs.artifact }}:${{ steps.set-variables.outputs.version }}-${{ steps.set-variables.outputs.sha }} \
            -t ghcr.io/firezone/${{ steps.set-variables.outputs.artifact }}:${{ steps.set-variables.outputs.major_version }} \
            -t ghcr.io/firezone/${{ steps.set-variables.outputs.artifact }}:${{ steps.set-variables.outputs.major_minor_version }} \
            -t ghcr.io/firezone/${{ steps.set-variables.outputs.artifact }}:latest \
            $SOURCE_TAG
-      - name: Copy Google Cloud Storage binaries to "latest" version
-        run: |
-          set -xe
-
-          ARCHITECTURES=(x86_64 aarch64 armv7)
-
-          for arch in "${ARCHITECTURES[@]}"; do
-            # Copy sha256sum.txt
-            gcloud storage cp \
-              "gs://firezone-staging-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.sha }}/$arch.sha256sum.txt" \
-              "gs://firezone-prod-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/latest/$arch.sha256sum.txt"
-            gcloud storage cp \
-              "gs://firezone-staging-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.sha }}/$arch.sha256sum.txt" \
-              "gs://firezone-prod-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.sha }}/$arch.sha256sum.txt"
-            gcloud storage cp \
-              "gs://firezone-staging-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.sha }}/$arch.sha256sum.txt" \
-              "gs://firezone-prod-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.version }}/$arch.sha256sum.txt"
-
-            # Copy binaries
-            gcloud storage cp \
-              "gs://firezone-staging-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.sha }}/$arch" \
-              "gs://firezone-prod-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/latest/$arch"
-            gcloud storage cp \
-              "gs://firezone-staging-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.sha }}/$arch" \
-              "gs://firezone-prod-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.sha }}/$arch"
-            gcloud storage cp \
-              "gs://firezone-staging-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.sha }}/$arch" \
-              "gs://firezone-prod-artifacts/firezone-${{ steps.set-variables.outputs.artifact }}/${{ steps.set-variables.outputs.version }}/$arch"
-          done

  create-publish-pr:
    runs-on: ubuntu-latest-xlarge