Production environment (#2449)

2026-01-27 10:18:54 +00:00 · 2023-10-19 19:20:51 -06:00
parent 919b7890e6
commit 66302a5063
23 changed files with 1614 additions and 121 deletions
--- a/.github/actions/setup-rust/action.yml
+++ b/.github/actions/setup-rust/action.yml
@@ -14,7 +14,7 @@ runs:
        workload_identity_provider: "projects/397012414171/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions"
        service_account: "github-actions@github-iam-387915.iam.gserviceaccount.com"
    - run: |
-        echo "SCCACHE_GCS_BUCKET=firezone-sccache" >> $GITHUB_ENV
+        echo "SCCACHE_GCS_BUCKET=firezone-staging-sccache" >> $GITHUB_ENV
        echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> $GITHUB_ENV
      shell: bash
    - uses: mozilla-actions/sccache-action@v0.0.3
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -63,3 +63,7 @@ updates:
    directory: terraform/environments/staging/
    schedule:
      interval: monthly
+  - package-ecosystem: terraform
+    directory: terraform/environments/production/
+    schedule:
+      interval: monthly
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -100,12 +100,12 @@ jobs:
          build-args: ${{ matrix.build-args }}
          context: ${{ matrix.context }}/
          cache-from: |
-            type=registry,ref=${{ steps.login.outputs.registry }}/firezone/cache/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
-            type=registry,ref=${{ steps.login.outputs.registry }}/firezone/cache/${{ matrix.image_name }}:main
+            type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.image_name }}:${{ env.CACHE_TAG }}
+            type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.image_name }}:main
          # This will write the cache on main even if integration tests fail,
          # but it'll just be corrected on the next successful build.
          cache-to: |
-            type=registry,ref=${{steps.login.outputs.registry}}/firezone/cache/${{ matrix.image_name}}:${{ env.CACHE_TAG }}
+            type=registry,ref=${{steps.login.outputs.registry}}/cache/${{ matrix.image_name}}:${{ env.CACHE_TAG }}
          file: ${{ matrix.context }}/Dockerfile
          push: true
          target: ${{ matrix.target }}
--- a/.tool-versions
+++ b/.tool-versions
@@ -3,7 +3,7 @@
 nodejs 18.16.0
 elixir 1.15.6-otp-26
 erlang 26.1.1
-terraform 1.6.1
+terraform 1.6.2

 # Used for static analysis
 python 3.9.13
--- a/terraform/environments/production/.terraform.lock.hcl
+++ b/terraform/environments/production/.terraform.lock.hcl
@@ -0,0 +1,100 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/google" {
+  version     = "5.2.0"
+  constraints = "~> 5.2"
+  hashes = [
+    "h1:psy0RRnGgKCsDKjdXCxQMKt4A1BlcbspWLB5UZK3A5U=",
+    "zh:1d4c5b154d4764a0e3e8893193dc71ba5a4cdb2d9d9dd20f69312cc75399b038",
+    "zh:26c5c6ad5edc27c643f43d950ffe982267b732723a09fef74c672ede7a7459f7",
+    "zh:2b48824692ecc7fe8ae3366010a7cf8b441aa2ecb4b6e9777638952844eff19e",
+    "zh:2f77cbb0528e58228117c7976e8864e7604614123c8b33d7329ffb0d084505b9",
+    "zh:408e6a680c4b7235dc677b8ba6ccbda0bf07ffcbd3d13767474eea2c5177488f",
+    "zh:68c2e914cf71ff490b4dbc6487900c35f702285cb0047614eccafb6ff057b748",
+    "zh:849052c81c2ea4c703b22af9ae524d3f45e42c7e9a3553c1ff7a95f49fde6886",
+    "zh:8f764a4ddcd5eea9f81cc72bb2fd29e2549a91b66faf8df8583c584298a26a86",
+    "zh:dddc597b4af5e2dc772ec4291e39daffb4dc46f2cccde1d3a6d2cbe8d291743d",
+    "zh:de9752d744bd91fd35e589fea0d8a72f983fe6fc872cfd19841758dcb8629a3b",
+    "zh:ec40d112e5022e2ba408bdfab1fd2d4f30c0183db02a771fdf26cd3a8c7e9949",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/google-beta" {
+  version     = "5.2.0"
+  constraints = "~> 5.2"
+  hashes = [
+    "h1:lPgmiosn4AzF9x1p+Xf0Pva1mlOWo2VL/rXr49Rcmv0=",
+    "zh:09834404dd19d9191d29a2a58a6838ecd9f70b54e24e39c75f3063d4345671e5",
+    "zh:34c8564245834b2f8a2ed7e70880b1553e79dd55083cdadc0791fb2f611cd5a1",
+    "zh:7ee42223685859efea71bfe90c2b0e37a1ef0b79d523c415fa4196c307dc6024",
+    "zh:898c2038a828dce2a5acd497a60dd1074cd06f7c45a3b17f3fb7351d61f87f11",
+    "zh:8d46416318f51f38291724af1c0758abc45424c5f6e76e7405da6284e017e23a",
+    "zh:a1cb9ddf8f9aca6970efb349cf70fcffa051eb73675137d585a1da857ed2d47f",
+    "zh:b2f906c9261c7d9ce978f0f2915e26c829a363358d49c788e6cdbb3ebe7965b9",
+    "zh:b35f0c6167860f2014287336bc52f3c9cd6f5c5e0a1488ac76c2f8dc929077d4",
+    "zh:b6157769ad44e7e31aed666cdda5cd7e697ec422e0b94c28ba88a393fa5abfbe",
+    "zh:e15b215803700dae0ff68545f66c56a596e399ad3ecb1a76a924d9bc185b4385",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+    "zh:f9c8fac2a1500b647c9fab69cb5e29c2207c0f1682e497deaf0007ed226e8e4b",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/null" {
+  version = "3.2.1"
+  hashes = [
+    "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=",
+    "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840",
+    "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb",
+    "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5",
+    "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238",
+    "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc",
+    "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970",
+    "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2",
+    "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5",
+    "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f",
+    "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+  version = "3.5.1"
+  hashes = [
+    "h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=",
+    "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64",
+    "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d",
+    "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831",
+    "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3",
+    "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b",
+    "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2",
+    "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865",
+    "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03",
+    "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602",
+    "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/tls" {
+  version     = "4.0.4"
+  constraints = "~> 4.0"
+  hashes = [
+    "h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=",
+    "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55",
+    "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848",
+    "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be",
+    "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5",
+    "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe",
+    "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e",
+    "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48",
+    "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8",
+    "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60",
+    "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e",
+    "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}
--- a/terraform/environments/production/dns.tf
+++ b/terraform/environments/production/dns.tf
@@ -0,0 +1,438 @@
+# Allow Google Cloud and Let's Encrypt to issue certificates for our domain
+resource "google_dns_record_set" "dns-caa" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type = "CAA"
+  name = module.google-cloud-dns.dns_name
+  rrdatas = [
+    "0 issue \"letsencrypt.org\"",
+    "0 issue \"pki.goog\"",
+    "0 iodef \"mailto:security@firezone.dev\""
+  ]
+  ttl = 3600
+}
+
+# Website
+
+# Vercel doesn't support IPv6
+# resource "google_dns_record_set" "website-ipv6" {
+#   project      = module.google-cloud-project.project.project_id
+#   managed_zone = module.google-cloud-dns.zone_name
+
+#   type    = "AAAA"
+#   name    = module.google-cloud-dns.dns_name
+#   rrdatas = ["2001:19f0:ac02:bb:5400:4ff:fe47:6bdf"]
+#   ttl     = 3600
+# }
+
+resource "google_dns_record_set" "website-ipv4" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "A"
+  name    = module.google-cloud-dns.dns_name
+  rrdatas = ["76.76.21.21"]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "website-www-redirect" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "www.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["cname.vercel-dns.com."]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "status-page" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "status.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["bs4nszn1hdh6.stspg-customer.com."]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "blog-ipv4" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "A"
+  name    = "blog.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["45.63.84.183"]
+  ttl     = 3600
+}
+resource "google_dns_record_set" "blog-ipv6" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "AAAA"
+  name    = "blog.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["2001:19f0:ac02:bb:5400:4ff:fe47:6bdf"]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "docs-ipv4" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "A"
+  name    = "docs.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["45.63.84.183"]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "docs-ipv6" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "AAAA"
+  name    = "docs.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["2001:19f0:ac02:bb:5400:4ff:fe47:6bdf"]
+  ttl     = 3600
+}
+
+## TODO: get rid off this one
+resource "google_dns_record_set" "awsdemo-ipv4" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "A"
+  name    = "awsdemo.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["52.200.241.107"]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "awsdemo-acme-verification" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "TXT"
+  name    = "_acme-challenge.awsdemo.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["sX54Me2woKpf_iLC4R9Il_8U8OuMTtGqRXOo5fveCNU"]
+  ttl     = 3600
+}
+
+## TODO: get rid off this one
+resource "google_dns_record_set" "docker-dev-ipv4" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "A"
+  name    = "docker-dev.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["3.101.147.119"]
+  ttl     = 3600
+}
+
+# Third-party services
+
+## Sendgrid
+resource "google_dns_record_set" "sendgrid-project" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "23539796.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["sendgrid.net."]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "sendgrid-return-1" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "em8227.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["u23539796.wl047.sendgrid.net."]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "sendgrid-return-2" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "url6320.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["sendgrid.net."]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "sendgrid-domainkey1" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "s1._domainkey.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["s1.domainkey.u23539796.wl047.sendgrid.net."]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "sendgrid-domainkey2" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "s2._domainkey.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["s2.domainkey.u23539796.wl047.sendgrid.net."]
+  ttl     = 3600
+}
+
+# Postmark
+
+resource "google_dns_record_set" "postmark-dkim" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = "20230606183724pm._domainkey.${module.google-cloud-dns.dns_name}"
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "k=rsa;p=k=rsa;p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQClXI0pMLt49Ib2jTQ3bCIw1QtEySHuaaOzk3Li0c9R3xAuOtt2PcxNx1TEgIdOA7fw6ONN1YyPf68NXOw7J3dV1Ldfln6VxRYcXaPSqhNtftaK87Rr6VqiJRiP4iEYQi4IQa9JJ4Za6s/aSLmji5mob7u3iI/Bj412Krkao6wLwwIDAQAB"
+  ]
+}
+
+resource "google_dns_record_set" "postmark-return" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "pm-bounces.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["pm.mtasv.net."]
+  ttl     = 3600
+}
+
+# GitHub
+
+resource "google_dns_record_set" "github-verification" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = "_github-challenge-firezone-organization.${module.google-cloud-dns.dns_name}"
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "ca4903847a"
+  ]
+}
+
+# Twilio
+
+resource "google_dns_record_set" "twilio-verification" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = "_twilio.${module.google-cloud-dns.dns_name}"
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "twilio-domain-verification=12fc8b0170bb9b63e4b6de67a5c923f0"
+  ]
+}
+
+# Google Workspace
+
+resource "google_dns_record_set" "google-verification" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = module.google-cloud-dns.dns_name
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    # TODO: only keep the last one needed
+    "google-site-verification=hbBLPfTlejIaxyFTPZN0RaIk6Y6qhQTG2yma7I06Emo",
+    "google-site-verification=oAugt2Arr7OyWaqJ0bkytkmIE-VQ8D_IFa-rdNiqa8s",
+    "google-site-verification=VDl82gbqVHJW6un8Mcki6qDhL_OGK6G8ByOB6qhaVbg",
+    "protonmail-verification=775efd155d2dec59fc6341d6bbfec288038f1917",
+  ]
+}
+
+resource "google_dns_record_set" "google-mail" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = module.google-cloud-dns.dns_name
+  type = "MX"
+  ttl  = 3600
+
+  rrdatas = [
+    "1 aspmx.l.google.com.",
+    "5 alt1.aspmx.l.google.com.",
+    "5 alt2.aspmx.l.google.com.",
+    "10 alt3.aspmx.l.google.com.",
+    "10 alt4.aspmx.l.google.com."
+  ]
+}
+
+resource "google_dns_record_set" "google-dmark" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+
+  name = "_dmarc.${module.google-cloud-dns.dns_name}"
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "\"v=DMARC1;\" \"p=reject;\" \"rua=mailto:dmarc-reports@firezone.dev;\" \"pct=100;\" \"adkim=r;\" \"aspf=r\""
+  ]
+}
+
+resource "google_dns_record_set" "google-spf" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = module.google-cloud-dns.dns_name
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "\"v=spf1 mx include:23723443.spf07.hubspotemail.net include:sendgrid.net include:_spf.google.com ~all\""
+  ]
+}
+
+resource "google_dns_record_set" "google-dkim" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = "20190728104345pm._domainkey.${module.google-cloud-dns.dns_name}"
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "\"v=DKIM1;\" \"k=rsa;\" \"p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAi1bjDNWHAhpLro2nw6WJ4Ye+JyA0gsMLHx1g+oS\" \"uGC6V0zo0Ftdt/tgvieaWbArClrz7Ce8986mih1P6iEESehTSarDrLlHPstIEI6UnjP7sAuIZtRsIrUI4NJM0Jg96uS4ezxIza3bzNxk3atMp0laCt+\" \"tbCeGLCPt4r9aygWIT/CRuNHZUm3CVwemN0celflXZF+FEg+mEJrkekasNtVJJ//XAdimvwe9CWOF/VoC+ZP0ocac3CFzng7NzSqYnCiaAZqJ3Pss0ueq0K/kqUxy8vh25Kd\" \"gyvdHSWdgnMFD251I/TBueScPZoUmo3ueYqwKxmW1J1uCkVx4NQ1xK2QIDAQAB"
+  ]
+}
+
+## ext. domain email server
+## TODO: get rid off this
+resource "google_dns_record_set" "google-ext-verification" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = "ext.${module.google-cloud-dns.dns_name}"
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "google-site-verification=xlFwz_eC6ksZ1dAJKwNzFISlZRpFRQ2mggo851altmI"
+  ]
+}
+
+resource "google_dns_record_set" "google-ext-mail" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = "ext.${module.google-cloud-dns.dns_name}"
+
+  type = "MX"
+  ttl  = 3600
+
+  rrdatas = [
+    "1 aspmx.l.google.com.",
+    "5 alt1.aspmx.l.google.com.",
+    "5 alt2.aspmx.l.google.com.",
+    "10 alt3.aspmx.l.google.com.",
+    "10 alt4.aspmx.l.google.com."
+  ]
+}
+
+resource "google_dns_record_set" "google-ext-dmark" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+
+  name = "_dmarc.ext.${module.google-cloud-dns.dns_name}"
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "\"v=DMARC1;\" \"p=reject;\" \"rua=mailto:dmarc-reports@firezone.dev;\" \"pct=100;\" \"adkim=s;\" \"aspf=s\""
+  ]
+}
+
+resource "google_dns_record_set" "google-ext-spf" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = "ext.${module.google-cloud-dns.dns_name}"
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "\"v=spf1 include:_spf.google.com ~all\""
+  ]
+}
+
+resource "google_dns_record_set" "google-ext-dkim" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  name = "20190728104345pm._domainkey.ext.${module.google-cloud-dns.dns_name}"
+  type = "TXT"
+  ttl  = 3600
+
+  rrdatas = [
+    "\"v=DKIM1;\" \"k=rsa;\" \"p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAubhkd+M9O2fILLpfRzCN5vhd81uSfaCbfeQ5Uf/BsBnuJ8AYOsyW\" \"bzy3UYU1y2JnJi1D8U+o1idcTPC1wB1okBHUnohI1O9hRDHb5NzV4NTxK0D36ESbgGzv94xu1n1GfxoO/wWga69eu/unz79/SRdVEida09bF0eXg9q\" \"5dtyIPI9NvYGtKAvLIABYHkutlUA2dNggraVTXldTlccMWmtd9uzemBg0bpN6zxygSLM9PSsEf0WEJJYvUXrEIQI4o9Ujh1/PqIgRpdqRAbmyhO3BobGNm5qmn3i1ZxWF0L\" \"T8zC3QShMPO+BagJlDav1ZNxBtih+vqqeyJvm8gwPXHiQIDAQAB"
+  ]
+}
+
+# HubSpot
+resource "google_dns_record_set" "hubspot-domainkey1" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "hs1-23723443._domainkey.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["firezone-dev.hs07a.dkim.hubspotemail.net."]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "hubspot-domainkey2" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "hs2-23723443._domainkey.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["firezone-dev.hs07b.dkim.hubspotemail.net."]
+  ttl     = 3600
+}
+
+# Proton
+## TODO: get rid off this
+resource "google_dns_record_set" "proton-domainkey1" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "protonmail._domainkey.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["protonmail.domainkey.dbmieophzl5yorultqalvxh5cjl65qstyplotj4asfsqiqan6337a.domains.proton.ch."]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "proton-domainkey2" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "protonmail2._domainkey.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["protonmail2.domainkey.dbmieophzl5yorultqalvxh5cjl65qstyplotj4asfsqiqan6337a.domains.proton.ch."]
+  ttl     = 3600
+}
+
+resource "google_dns_record_set" "proton-domainkey3" {
+  project      = module.google-cloud-project.project.project_id
+  managed_zone = module.google-cloud-dns.zone_name
+
+  type    = "CNAME"
+  name    = "protonmail3._domainkey.${module.google-cloud-dns.dns_name}"
+  rrdatas = ["protonmail3.domainkey.dbmieophzl5yorultqalvxh5cjl65qstyplotj4asfsqiqan6337a.domains.proton.ch."]
+  ttl     = 3600
+}
--- a/terraform/environments/production/main.tf
+++ b/terraform/environments/production/main.tf
@@ -0,0 +1,723 @@
+locals {
+  project_owners = [
+    "a@firezone.dev",
+    "bmanifold@firezone.dev",
+    "gabriel@firezone.dev",
+    "jamil@firezone.dev",
+    "thomas@firezone.dev"
+  ]
+
+  region            = "us-east1"
+  availability_zone = "us-east1-d"
+
+  tld = "firezone.dev"
+}
+
+terraform {
+  cloud {
+    organization = "firezone"
+    hostname     = "app.terraform.io"
+
+    workspaces {
+      name = "production"
+    }
+  }
+}
+
+provider "random" {}
+provider "null" {}
+provider "google" {}
+provider "google-beta" {}
+
+# Create the project
+module "google-cloud-project" {
+  source = "../../modules/google-cloud-project"
+
+  id                 = "firezone-production"
+  name               = "Production Environment"
+  organization_id    = "335836213177"
+  billing_account_id = "01DFC9-3D6951-579BE1"
+}
+
+# Grant owner access to the project
+resource "google_project_iam_binding" "project_owners" {
+  project = module.google-cloud-project.project.project_id
+  role    = "roles/owner"
+  members = formatlist("user:%s", local.project_owners)
+}
+
+# Grant GitHub Actions ability to write to the container registry
+module "google-artifact-registry" {
+  source = "../../modules/google-artifact-registry"
+
+  project_id   = module.google-cloud-project.project.project_id
+  project_name = module.google-cloud-project.name
+
+  region = local.region
+
+  immutable_tags = true
+
+  writers = [
+    # This is GitHub Actions service account configured manually
+    # in the project github-iam-387915
+    "serviceAccount:github-actions@github-iam-387915.iam.gserviceaccount.com"
+  ]
+}
+
+# Create a VPC
+module "google-cloud-vpc" {
+  source = "../../modules/google-cloud-vpc"
+
+  project_id = module.google-cloud-project.project.project_id
+  name       = module.google-cloud-project.project.project_id
+
+  nat_region = local.region
+}
+
+# Enable Google Cloud Storage for the project
+module "google-cloud-storage" {
+  source = "../../modules/google-cloud-storage"
+
+  project_id = module.google-cloud-project.project.project_id
+}
+
+# Create DNS managed zone
+module "google-cloud-dns" {
+  source = "../../modules/google-cloud-dns"
+
+  project_id = module.google-cloud-project.project.project_id
+
+  tld            = local.tld
+  dnssec_enabled = false
+}
+
+# Create the Cloud SQL database
+module "google-cloud-sql" {
+  source     = "../../modules/google-cloud-sql"
+  project_id = module.google-cloud-project.project.project_id
+  network    = module.google-cloud-vpc.id
+
+  compute_region            = local.region
+  compute_availability_zone = local.availability_zone
+
+  compute_instance_cpu_count   = "2"
+  compute_instance_memory_size = "7680"
+
+  database_name = module.google-cloud-project.project.project_id
+
+  database_highly_available = true
+  database_backups_enabled  = true
+
+  database_read_replica_locations = []
+
+  database_flags = {
+    # Increase the connections count a bit, but we need to set it to Ecto ((pool_count * pool_size) + 50)
+    "max_connections" = "500"
+
+    # Sets minimum threshold on dead tuples to prevent autovaccum running too often on small tables
+    # where 5% is less than 50 records
+    "autovacuum_vacuum_threshold" = "50"
+
+    # Trigger autovaccum for every 5% of the table changed
+    "autovacuum_vacuum_scale_factor"  = "0.05"
+    "autovacuum_analyze_scale_factor" = "0.05"
+
+    # Give autovacuum 4x the cost limit to prevent it from never finishing
+    # on big tables
+    "autovacuum_vacuum_cost_limit" = "800"
+
+    # Give hash joins a bit more memory to work with
+    # "hash_mem_multiplier" = "3"
+
+    # This is standard value for work_mem
+    "work_mem" = "4096"
+  }
+}
+
+# Generate secrets
+resource "random_password" "erlang_cluster_cookie" {
+  length  = 64
+  special = false
+}
+
+resource "random_password" "auth_token_key_base" {
+  length  = 64
+  special = false
+}
+
+resource "random_password" "auth_token_salt" {
+  length  = 32
+  special = false
+}
+
+resource "random_password" "relays_auth_token_key_base" {
+  length  = 64
+  special = false
+}
+
+resource "random_password" "relays_auth_token_salt" {
+  length  = 32
+  special = false
+}
+
+resource "random_password" "gateways_auth_token_key_base" {
+  length  = 64
+  special = false
+}
+
+resource "random_password" "gateways_auth_token_salt" {
+  length  = 32
+  special = false
+}
+
+resource "random_password" "secret_key_base" {
+  length  = 64
+  special = false
+}
+
+resource "random_password" "live_view_signing_salt" {
+  length  = 32
+  special = false
+}
+
+resource "random_password" "cookie_signing_salt" {
+  length  = 32
+  special = false
+}
+
+resource "random_password" "cookie_encryption_salt" {
+  length  = 32
+  special = false
+}
+
+# Create VPC subnet for the application instances,
+# we want all apps to be in the same VPC in order for Erlang clustering to work
+resource "google_compute_subnetwork" "apps" {
+  project = module.google-cloud-project.project.project_id
+
+  name = "app"
+
+  stack_type = "IPV4_IPV6"
+
+  ip_cidr_range = "10.128.0.0/20"
+  region        = local.region
+  network       = module.google-cloud-vpc.id
+
+  ipv6_access_type = "EXTERNAL"
+
+  private_ip_google_access = true
+}
+
+# Create SQL user and database
+resource "random_password" "firezone_db_password" {
+  length = 16
+}
+
+resource "google_sql_user" "firezone" {
+  project = module.google-cloud-project.project.project_id
+
+  instance = module.google-cloud-sql.master_instance_name
+
+  name     = "firezone"
+  password = random_password.firezone_db_password.result
+}
+
+resource "google_sql_database" "firezone" {
+  project = module.google-cloud-project.project.project_id
+
+  name     = "firezone"
+  instance = module.google-cloud-sql.master_instance_name
+}
+
+# Create bucket for client logs
+resource "google_storage_bucket" "client-logs" {
+  project = module.google-cloud-project.project.project_id
+  name    = "${module.google-cloud-project.project.project_id}-client-logs"
+
+  location = "US"
+
+  lifecycle_rule {
+    condition {
+      age = 3
+    }
+
+    action {
+      type = "Delete"
+    }
+  }
+
+  lifecycle_rule {
+    condition {
+      age = 1
+    }
+
+    action {
+      type = "AbortIncompleteMultipartUpload"
+    }
+  }
+
+  logging {
+    log_bucket        = true
+    log_object_prefix = "firezone.dev/clients"
+  }
+
+  public_access_prevention    = "enforced"
+  uniform_bucket_level_access = true
+
+  lifecycle {
+    prevent_destroy = true
+    ignore_changes  = []
+  }
+}
+
+locals {
+  cluster = {
+    name   = "firezone"
+    cookie = base64encode(random_password.erlang_cluster_cookie.result)
+  }
+
+  shared_application_environment_variables = [
+    # Database
+    {
+      name  = "DATABASE_HOST"
+      value = module.google-cloud-sql.master_instance_ip_address
+    },
+    {
+      name  = "DATABASE_NAME"
+      value = google_sql_database.firezone.name
+    },
+    {
+      name  = "DATABASE_USER"
+      value = google_sql_user.firezone.name
+    },
+    {
+      name  = "DATABASE_PASSWORD"
+      value = google_sql_user.firezone.password
+    },
+    # Secrets
+    {
+      name  = "SECRET_KEY_BASE"
+      value = random_password.secret_key_base.result
+    },
+    {
+      name  = "AUTH_TOKEN_KEY_BASE"
+      value = base64encode(random_password.auth_token_key_base.result)
+    },
+    {
+      name  = "AUTH_TOKEN_SALT"
+      value = base64encode(random_password.auth_token_salt.result)
+    },
+    {
+      name  = "RELAYS_AUTH_TOKEN_KEY_BASE"
+      value = base64encode(random_password.relays_auth_token_key_base.result)
+    },
+    {
+      name  = "RELAYS_AUTH_TOKEN_SALT"
+      value = base64encode(random_password.relays_auth_token_salt.result)
+    },
+    {
+      name  = "GATEWAYS_AUTH_TOKEN_KEY_BASE"
+      value = base64encode(random_password.gateways_auth_token_key_base.result)
+    },
+    {
+      name  = "GATEWAYS_AUTH_TOKEN_SALT"
+      value = base64encode(random_password.gateways_auth_token_salt.result)
+    },
+    {
+      name  = "SECRET_KEY_BASE"
+      value = base64encode(random_password.secret_key_base.result)
+    },
+    {
+      name  = "LIVE_VIEW_SIGNING_SALT"
+      value = base64encode(random_password.live_view_signing_salt.result)
+    },
+    {
+      name  = "COOKIE_SIGNING_SALT"
+      value = base64encode(random_password.cookie_signing_salt.result)
+    },
+    {
+      name  = "COOKIE_ENCRYPTION_SALT"
+      value = base64encode(random_password.cookie_encryption_salt.result)
+    },
+    # Erlang
+    {
+      name  = "ERLANG_DISTRIBUTION_PORT"
+      value = "9000"
+    },
+    {
+      name  = "CLUSTER_NAME"
+      value = local.cluster.name
+    },
+    {
+      name  = "ERLANG_CLUSTER_ADAPTER"
+      value = "Elixir.Domain.Cluster.GoogleComputeLabelsStrategy"
+    },
+    {
+      name = "ERLANG_CLUSTER_ADAPTER_CONFIG"
+      value = jsonencode({
+        project_id          = module.google-cloud-project.project.project_id
+        cluster_name        = local.cluster.name
+        cluster_name_label  = "cluster_name"
+        node_name_label     = "application"
+        polling_interval_ms = 7000
+      })
+    },
+    {
+      name  = "RELEASE_COOKIE"
+      value = local.cluster.cookie
+    },
+    # Auth
+    {
+      name  = "AUTH_PROVIDER_ADAPTERS"
+      value = "email,openid_connect,google_workspace,token"
+    },
+    # Telemetry
+    {
+      name  = "TELEMETRY_ENABLED"
+      value = "false"
+    },
+    {
+      name  = "INSTRUMENTATION_CLIENT_LOGS_ENABLED"
+      value = true
+    },
+    {
+      name  = "INSTRUMENTATION_CLIENT_LOGS_BUCKET"
+      value = google_storage_bucket.client-logs.name
+    },
+    # Emails
+    {
+      name  = "OUTBOUND_EMAIL_ADAPTER"
+      value = "Elixir.Swoosh.Adapters.Postmark"
+    },
+    {
+      name  = "OUTBOUND_EMAIL_FROM"
+      value = "support@firezone.dev"
+    },
+    {
+      name  = "OUTBOUND_EMAIL_ADAPTER_OPTS"
+      value = "{\"api_key\":\"${var.postmark_server_api_token}\"}"
+    }
+  ]
+}
+
+module "web" {
+  source     = "../../modules/elixir-app"
+  project_id = module.google-cloud-project.project.project_id
+
+  compute_instance_type               = "n1-standard-1"
+  compute_instance_region             = local.region
+  compute_instance_availability_zones = ["${local.region}-d"]
+
+  dns_managed_zone_name = module.google-cloud-dns.zone_name
+
+  vpc_network    = module.google-cloud-vpc.self_link
+  vpc_subnetwork = google_compute_subnetwork.apps.self_link
+
+  container_registry = module.google-artifact-registry.url
+
+  image_repo = module.google-artifact-registry.repo
+  image      = "web"
+  image_tag  = var.web_image_tag
+
+  scaling_horizontal_replicas = 2
+
+  observability_log_level = "debug"
+
+  erlang_release_name   = "firezone"
+  erlang_cluster_cookie = random_password.erlang_cluster_cookie.result
+
+  application_name    = "web"
+  application_version = replace(var.web_image_tag, ".", "-")
+
+  application_dns_tld = "app.${local.tld}"
+
+  application_ports = [
+    {
+      name     = "http"
+      protocol = "TCP"
+      port     = 8080
+
+      health_check = {
+        initial_delay_sec = 60
+
+        check_interval_sec  = 15
+        timeout_sec         = 10
+        healthy_threshold   = 1
+        unhealthy_threshold = 2
+
+        http_health_check = {
+          request_path = "/healthz"
+        }
+      }
+    }
+  ]
+
+  application_environment_variables = concat([
+    # Web Server
+    {
+      name  = "EXTERNAL_URL"
+      value = "https://app.${local.tld}"
+    },
+    {
+      name  = "PHOENIX_HTTP_WEB_PORT"
+      value = "8080"
+    }
+  ], local.shared_application_environment_variables)
+
+  application_labels = {
+    "cluster_name" = local.cluster.name
+  }
+}
+
+module "api" {
+  source     = "../../modules/elixir-app"
+  project_id = module.google-cloud-project.project.project_id
+
+  compute_instance_type               = "n1-standard-1"
+  compute_instance_region             = local.region
+  compute_instance_availability_zones = ["${local.region}-d"]
+
+  dns_managed_zone_name = module.google-cloud-dns.zone_name
+
+  vpc_network    = module.google-cloud-vpc.self_link
+  vpc_subnetwork = google_compute_subnetwork.apps.self_link
+
+  container_registry = module.google-artifact-registry.url
+
+  image_repo = module.google-artifact-registry.repo
+  image      = "api"
+  image_tag  = var.api_image_tag
+
+  scaling_horizontal_replicas = 2
+
+  observability_log_level = "debug"
+
+  erlang_release_name   = "firezone"
+  erlang_cluster_cookie = random_password.erlang_cluster_cookie.result
+
+  application_name    = "api"
+  application_version = replace(var.api_image_tag, ".", "-")
+
+  application_dns_tld = "api.${local.tld}"
+
+  application_ports = [
+    {
+      name     = "http"
+      protocol = "TCP"
+      port     = 8080
+
+      health_check = {
+        initial_delay_sec = 60
+
+        check_interval_sec  = 15
+        timeout_sec         = 10
+        healthy_threshold   = 1
+        unhealthy_threshold = 3
+
+        http_health_check = {
+          request_path = "/healthz"
+        }
+      }
+    }
+  ]
+
+  application_environment_variables = concat([
+    # Web Server
+    {
+      name  = "EXTERNAL_URL"
+      value = "https://api.${local.tld}"
+    },
+    {
+      name  = "PHOENIX_HTTP_API_PORT"
+      value = "8080"
+    },
+  ], local.shared_application_environment_variables)
+
+  application_labels = {
+    "cluster_name" = local.cluster.name
+  }
+
+  application_token_scopes = [
+    "https://www.googleapis.com/auth/cloud-platform"
+  ]
+}
+
+## Allow API nodes to sign URLs for Google Cloud Storage
+resource "google_storage_bucket_iam_member" "sign-urls" {
+  bucket = google_storage_bucket.client-logs.name
+  role   = "roles/storage.objectAdmin"
+  member = "serviceAccount:${module.api.service_account.email}"
+}
+
+resource "google_project_iam_custom_role" "sign-urls" {
+  project = module.google-cloud-project.project.project_id
+
+  title = "Sign URLs for Google Cloud Storage"
+
+  role_id = "iam.sign_urls"
+
+  permissions = [
+    "iam.serviceAccounts.signBlob"
+  ]
+}
+
+resource "google_project_iam_member" "sign-urls" {
+  project = module.google-cloud-project.project.project_id
+  role    = "projects/${module.google-cloud-project.project.project_id}/roles/${google_project_iam_custom_role.sign-urls.role_id}"
+  member  = "serviceAccount:${module.api.service_account.email}"
+}
+
+# Erlang Cluster
+## Allow traffic between Elixir apps for Erlang clustering
+resource "google_compute_firewall" "erlang-distribution" {
+  project = module.google-cloud-project.project.project_id
+
+  name    = "erlang-distribution"
+  network = module.google-cloud-vpc.self_link
+
+  allow {
+    protocol = "tcp"
+    ports    = [4369, 9000]
+  }
+
+  allow {
+    protocol = "udp"
+    ports    = [4369, 9000]
+  }
+
+  source_ranges = [google_compute_subnetwork.apps.ip_cidr_range]
+  target_tags   = concat(module.web.target_tags, module.api.target_tags)
+}
+
+## Allow service account to list running instances
+resource "google_project_iam_custom_role" "erlang-discovery" {
+  project = module.google-cloud-project.project.project_id
+
+  title       = "Read list of Compute instances"
+  description = "This role is used for Erlang Cluster discovery and allows to list running instances."
+
+  role_id = "compute.list_instances"
+  permissions = [
+    "compute.instances.list",
+    "compute.zones.list"
+  ]
+}
+
+resource "google_project_iam_member" "application" {
+  for_each = toset([
+    module.api.service_account.email,
+    module.web.service_account.email,
+  ])
+
+  project = module.google-cloud-project.project.project_id
+
+  role   = "projects/${module.google-cloud-project.project.project_id}/roles/${google_project_iam_custom_role.erlang-discovery.role_id}"
+  member = "serviceAccount:${each.value}"
+}
+
+# Deploy relays
+module "relays" {
+  count = var.relay_portal_token != null ? 1 : 0
+
+  source     = "../../modules/relay-app"
+  project_id = module.google-cloud-project.project.project_id
+
+  instances = {
+    "asia-east1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["asia-east1-a"]
+    }
+
+    "asia-south1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["asia-south1-a"]
+    }
+
+    "australia-southeast1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["australia-southeast1-a"]
+    }
+
+    "me-central1" = {
+      type     = "n2-standard-2"
+      replicas = 1
+      zones    = ["me-central1-a"]
+    }
+
+    "europe-west1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["europe-west1-d"]
+    }
+
+    "southamerica-east1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["southamerica-east1-b"]
+    }
+
+    "us-east1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["us-east1-d"]
+    }
+
+    "us-west2" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["us-west2-b"]
+    }
+
+    "us-central1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["us-central1-b"]
+    }
+  }
+
+  container_registry = module.google-artifact-registry.url
+
+  image_repo = module.google-artifact-registry.repo
+  image      = "relay"
+  image_tag  = var.relay_image_tag
+
+  observability_log_level = "debug,firezone_relay=trace,hyper=off,h2=warn,tower=warn,wire=trace"
+
+  application_name    = "relay"
+  application_version = replace(var.relay_image_tag, ".", "-")
+
+  health_check = {
+    name     = "health"
+    protocol = "TCP"
+    port     = 8080
+
+    initial_delay_sec = 60
+
+    check_interval_sec  = 15
+    timeout_sec         = 10
+    healthy_threshold   = 1
+    unhealthy_threshold = 3
+
+    http_health_check = {
+      request_path = "/healthz"
+    }
+  }
+
+  portal_websocket_url = "wss://api.${local.tld}"
+  portal_token         = var.relay_portal_token
+}
+
+module "ops" {
+  source = "../../modules/google-cloud-ops"
+
+  project_id = module.google-cloud-project.project.project_id
+
+  slack_alerts_auth_token = var.slack_alerts_auth_token
+  slack_alerts_channel    = var.slack_alerts_channel
+
+  api_host = module.api.host
+  web_host = module.web.host
+}
--- a/terraform/environments/production/outputs.tf
+++ b/terraform/environments/production/outputs.tf
@@ -0,0 +1,3 @@
+output "dns_name_servers" {
+  value = module.google-cloud-dns.name_servers
+}
--- a/terraform/environments/production/variables.tf
+++ b/terraform/environments/production/variables.tf
@@ -0,0 +1,34 @@
+variable "api_image_tag" {
+  type        = string
+  description = "Image tag for the api service"
+}
+
+variable "web_image_tag" {
+  type        = string
+  description = "Image tag for the web service"
+}
+
+variable "relay_image_tag" {
+  type        = string
+  description = "Image tag for the relay service"
+}
+
+variable "relay_portal_token" {
+  type    = string
+  default = null
+}
+
+variable "slack_alerts_channel" {
+  type        = string
+  description = "Slack channel which will receive monitoring alerts"
+  default     = "#feed-infra"
+}
+
+variable "slack_alerts_auth_token" {
+  type        = string
+  description = "Slack auth token for the infra alerts channel"
+}
+
+variable "postmark_server_api_token" {
+  type = string
+}
--- a/terraform/environments/production/versions.tf
+++ b/terraform/environments/production/versions.tf
@@ -0,0 +1,30 @@
+terraform {
+  required_version = "1.6.2"
+
+  required_providers {
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.5"
+    }
+
+    null = {
+      source  = "hashicorp/null"
+      version = "~> 3.2"
+    }
+
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 5.2"
+    }
+
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 5.2"
+    }
+
+    tls = {
+      source  = "hashicorp/tls"
+      version = "~> 4.0"
+    }
+  }
+}
--- a/terraform/environments/staging/ci.tf
+++ b/terraform/environments/staging/ci.tf
@@ -0,0 +1,123 @@
+# Docker layer caching
+resource "google_artifact_registry_repository" "cache" {
+  provider = google-beta
+  project  = module.google-cloud-project.project.project_id
+
+  location      = local.region
+  repository_id = "cache"
+  description   = "Repository for storing Docker images in the ${module.google-cloud-project.name}."
+
+  format = "DOCKER"
+
+  docker_config {
+    immutable_tags = false
+  }
+
+  cleanup_policies {
+    id     = "keep-latest-release"
+    action = "KEEP"
+
+    condition {
+      tag_state    = "TAGGED"
+      tag_prefixes = ["latest"]
+    }
+  }
+
+  cleanup_policies {
+    id     = "keep-minimum-versions"
+    action = "KEEP"
+
+    most_recent_versions {
+      keep_count = 5
+    }
+  }
+
+  cleanup_policies {
+    id     = "gc-untagged"
+    action = "DELETE"
+
+    condition {
+      tag_state  = "UNTAGGED"
+      older_than = "${14 * 24 * 60 * 60}s"
+    }
+  }
+
+  cleanup_policies {
+    id     = "gc-cache"
+    action = "DELETE"
+
+    condition {
+      tag_state  = "ANY"
+      older_than = "${30 * 24 * 60 * 60}s"
+    }
+  }
+
+  depends_on = [
+    module.google-artifact-registry
+  ]
+}
+
+data "google_iam_policy" "caches_policy" {
+  binding {
+    role    = "roles/artifactregistry.reader"
+    members = ["allUsers"]
+  }
+
+  binding {
+    role    = "roles/artifactregistry.writer"
+    members = local.ci_iam_members
+  }
+}
+
+resource "google_artifact_registry_repository_iam_policy" "policy" {
+  project    = google_artifact_registry_repository.cache.project
+  location   = google_artifact_registry_repository.cache.location
+  repository = google_artifact_registry_repository.cache.name
+
+  policy_data = data.google_iam_policy.caches_policy.policy_data
+}
+
+# sccache is used by Rust CI jobs
+resource "google_storage_bucket" "sccache" {
+  project = module.google-cloud-project.project.project_id
+  name    = "${module.google-cloud-project.project.project_id}-sccache"
+
+  location = "US"
+
+  lifecycle_rule {
+    condition {
+      age = 30
+    }
+
+    action {
+      type = "Delete"
+    }
+  }
+
+  lifecycle_rule {
+    condition {
+      age = 1
+    }
+
+    action {
+      type = "AbortIncompleteMultipartUpload"
+    }
+  }
+
+  public_access_prevention    = "inherited"
+  uniform_bucket_level_access = true
+}
+
+# resource "google_storage_bucket_iam_member" "public-sccache" {
+#   bucket = google_storage_bucket.sccache.name
+#   role   = "roles/storage.objectViewer"
+#   member = "allUsers"
+# }
+
+resource "google_storage_bucket_iam_member" "github-actions-sccache-access" {
+  for_each = toset(local.ci_iam_members)
+
+  bucket = google_storage_bucket.sccache.name
+  role   = "roles/storage.objectAdmin"
+  member = each.key
+}
--- a/terraform/environments/staging/dns.tf
+++ b/terraform/environments/staging/dns.tf
@@ -15,7 +15,7 @@ resource "google_dns_record_set" "dns-caa" {

 # Website

-resource "google_dns_record_set" "website-ipv4" {
+resource "google_dns_record_set" "website-ipv6" {
  project      = module.google-cloud-project.project.project_id
  managed_zone = module.google-cloud-dns.zone_name

@@ -25,7 +25,7 @@ resource "google_dns_record_set" "website-ipv4" {
  ttl     = 3600
 }

-resource "google_dns_record_set" "website-ipv6" {
+resource "google_dns_record_set" "website-ipv4" {
  project      = module.google-cloud-project.project.project_id
  managed_zone = module.google-cloud-dns.zone_name

--- a/terraform/environments/staging/main.tf
+++ b/terraform/environments/staging/main.tf
@@ -11,6 +11,12 @@ locals {
  availability_zone = "us-east1-d"

  tld = "firez.one"
+
+  # This is GitHub Actions service account configured manually
+  # in the project github-iam-387915
+  ci_iam_members = [
+    "serviceAccount:github-actions@github-iam-387915.iam.gserviceaccount.com"
+  ]
 }

 terraform {
@@ -57,13 +63,10 @@ module "google-artifact-registry" {

  immutable_tags = false

-  store_tagged_artifacts_for = "${90 * 24 * 60 * 60}s"
+  store_tagged_artifacts_for   = "${90 * 24 * 60 * 60}s"
+  store_untagged_artifacts_for = "${90 * 24 * 60 * 60}s"

-  writers = [
-    # This is GitHub Actions service account configured manually
-    # in the project github-iam-387915
-    "serviceAccount:github-actions@github-iam-387915.iam.gserviceaccount.com"
-  ]
+  writers = local.ci_iam_members
 }

 # Create a VPC
@@ -72,6 +75,8 @@ module "google-cloud-vpc" {

  project_id = module.google-cloud-project.project.project_id
  name       = module.google-cloud-project.project.project_id
+
+  nat_region = local.region
 }

 # Enable Google Cloud Storage for the project
@@ -190,12 +195,6 @@ resource "random_password" "cookie_encryption_salt" {
  special = false
 }

-# # Deploy nginx to the compute for HTTPS termination
-# # module "nginx" {
-# #   source = "../../modules/nginx"
-# #   project_id = module.google-cloud-project.project.project_id
-# # }
-
 # Create VPC subnet for the application instances,
 # we want all apps to be in the same VPC in order for Erlang clustering to work
 resource "google_compute_subnetwork" "apps" {
@@ -630,53 +629,53 @@ module "relays" {
  project_id = module.google-cloud-project.project.project_id

  instances = {
-    # "asia-east1" = {
-    #   type     = "n1-standard-1"
-    #   replicas = 1
-    #   zones    = ["asia-east1-a"]
-    # }
+    "asia-east1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["asia-east1-a"]
+    }

-    # "asia-south1" = {
-    #   type     = "n1-standard-1"
-    #   replicas = 1
-    #   zones    = ["asia-south1-a"]
-    # }
+    "asia-south1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["asia-south1-a"]
+    }

-    # "australia-southeast1" = {
-    #   type     = "n1-standard-1"
-    #   replicas = 1
-    #   zones    = ["australia-southeast1-a"]
-    # }
+    "australia-southeast1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["australia-southeast1-a"]
+    }

-    # "me-central1" = {
-    #   type     = "n2-standard-2"
-    #   replicas = 1
-    #   zones    = ["me-central1-a"]
-    # }
+    "me-central1" = {
+      type     = "n2-standard-2"
+      replicas = 1
+      zones    = ["me-central1-a"]
+    }

-    # "europe-west1" = {
-    #   type     = "n1-standard-1"
-    #   replicas = 1
-    #   zones    = ["europe-west1-d"]
-    # }
+    "europe-west1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["europe-west1-d"]
+    }

-    # "southamerica-east1" = {
-    #   type     = "n1-standard-1"
-    #   replicas = 1
-    #   zones    = ["southamerica-east1-b"]
-    # }
+    "southamerica-east1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["southamerica-east1-b"]
+    }

-    # "us-east1" = {
-    #   type     = "n1-standard-1"
-    #   replicas = 1
-    #   zones    = ["us-east1-d"]
-    # }
+    "us-east1" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["us-east1-d"]
+    }

-    # "us-west2" = {
-    #   type     = "n1-standard-1"
-    #   replicas = 1
-    #   zones    = ["us-west2-b"]
-    # }
+    "us-west2" = {
+      type     = "n1-standard-1"
+      replicas = 1
+      zones    = ["us-west2-b"]
+    }

    "us-central1" = {
      type     = "n1-standard-1"
@@ -821,3 +820,15 @@ resource "google_compute_firewall" "relays-ssh-ipv6" {
  source_ranges = ["::/0"]
  target_tags   = module.relays[0].target_tags
 }
+
+module "ops" {
+  source = "../../modules/google-cloud-ops"
+
+  project_id = module.google-cloud-project.project.project_id
+
+  slack_alerts_auth_token = var.slack_alerts_auth_token
+  slack_alerts_channel    = var.slack_alerts_channel
+
+  api_host = module.api.host
+  web_host = module.web.host
+}
--- a/terraform/environments/staging/nat.tf
+++ b/terraform/environments/staging/nat.tf
@@ -1,29 +0,0 @@
-## Router and Cloud NAT are required for instances without external IP address
-resource "google_compute_router" "default" {
-  project = module.google-cloud-project.project.project_id
-
-  name    = module.google-cloud-vpc.name
-  network = module.google-cloud-vpc.self_link
-  region  = local.region
-}
-
-resource "google_compute_router_nat" "application" {
-  project = module.google-cloud-project.project.project_id
-
-  name   = module.google-cloud-vpc.name
-  region = local.region
-
-  router = google_compute_router.default.name
-
-  nat_ip_allocate_option             = "AUTO_ONLY"
-  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
-
-  enable_dynamic_port_allocation = false
-  min_ports_per_vm               = 32
-
-  udp_idle_timeout_sec             = 30
-  icmp_idle_timeout_sec            = 30
-  tcp_established_idle_timeout_sec = 1200
-  tcp_transitory_idle_timeout_sec  = 30
-  tcp_time_wait_timeout_sec        = 120
-}
--- a/terraform/environments/staging/versions.tf
+++ b/terraform/environments/staging/versions.tf
@@ -1,5 +1,5 @@
 terraform {
-  required_version = "1.6.1"
+  required_version = "1.6.2"

  required_providers {
    random = {
--- a/terraform/modules/elixir-app/main.tf
+++ b/terraform/modules/elixir-app/main.tf
@@ -5,10 +5,8 @@ locals {
  application_labels = merge({
    managed_by = "terraform"

-    # Note: this labels are used to fetch a release name for Erlang Cluster,
-    # and filter then by version
+    # Note: this labels are used to fetch a release name for Erlang Cluster
    application = local.application_name
-    version     = local.application_version
  }, var.application_labels)

  application_environment_variables = concat([
@@ -149,8 +147,12 @@ resource "google_compute_instance_template" "application" {

  labels = merge({
    container-vm = data.google_compute_image.coreos.name
+
+    # This variable can be used by Erlang Cluster not to join nodes of older versions
+    version = local.application_version
  }, local.application_labels)

+
  scheduling {
    automatic_restart   = true
    on_host_maintenance = "MIGRATE"
--- a/terraform/modules/google-artifact-registry/main.tf
+++ b/terraform/modules/google-artifact-registry/main.tf
@@ -38,24 +38,17 @@ resource "google_artifact_registry_repository" "firezone" {
    }
  }

-  cleanup_policies {
-    id     = "gc-untagged"
-    action = "DELETE"
+  dynamic "cleanup_policies" {
+    for_each = var.store_untagged_artifacts_for != null ? [1] : []

-    condition {
-      tag_state  = "UNTAGGED"
-      older_than = "${90 * 24 * 60 * 60}s"
-    }
-  }
+    content {
+      id     = "gc-untagged"
+      action = "DELETE"

-  cleanup_policies {
-    id     = "gc-cache"
-    action = "DELETE"
-
-    condition {
-      tag_state             = "ANY"
-      package_name_prefixes = ["cache/"]
-      older_than            = "${30 * 24 * 60 * 60}s"
+      condition {
+        tag_state  = "UNTAGGED"
+        older_than = var.store_untagged_artifacts_for
+      }
    }
  }

--- a/terraform/modules/google-artifact-registry/variables.tf
+++ b/terraform/modules/google-artifact-registry/variables.tf
@@ -21,7 +21,13 @@ variable "immutable_tags" {
 }

 variable "store_tagged_artifacts_for" {
-  description = "Sets the maximum lifetime of artifacts, eg. `30d`. Keep empty to set to `null` to never delete them."
+  description = "Sets the maximum lifetime of artifacts, eg. `300s`. Keep empty to set to `null` to never delete them."
+  type        = string
+  default     = null
+}
+
+variable "store_untagged_artifacts_for" {
+  description = "Sets the maximum lifetime of artifacts, eg. `300s`. Keep empty to set to `null` to never delete them."
  type        = string
  default     = null
 }
--- a/terraform/modules/google-cloud-ops/main.tf
+++ b/terraform/modules/google-cloud-ops/main.tf
@@ -1,5 +1,5 @@
 resource "google_monitoring_notification_channel" "slack" {
-  project = module.google-cloud-project.project.project_id
+  project = var.project_id

  display_name = "Slack: #alerts-infra"
  type         = "slack"
@@ -14,7 +14,7 @@ resource "google_monitoring_notification_channel" "slack" {
 }

 resource "google_monitoring_uptime_check_config" "api-https" {
-  project = module.google-cloud-project.project.project_id
+  project = var.project_id

  display_name = "api-https"
  timeout      = "60s"
@@ -36,8 +36,8 @@ resource "google_monitoring_uptime_check_config" "api-https" {
    type = "uptime_url"

    labels = {
-      project_id = module.google-cloud-project.project.project_id
-      host       = module.api.host
+      project_id = var.project_id
+      host       = var.api_host
    }
  }

@@ -55,7 +55,7 @@ resource "google_monitoring_uptime_check_config" "api-https" {
 }

 resource "google_monitoring_uptime_check_config" "web-https" {
-  project = module.google-cloud-project.project.project_id
+  project = var.project_id

  display_name = "web-https"
  timeout      = "60s"
@@ -78,8 +78,8 @@ resource "google_monitoring_uptime_check_config" "web-https" {
    type = "uptime_url"

    labels = {
-      project_id = module.google-cloud-project.project.project_id
-      host       = module.web.host
+      project_id = var.project_id
+      host       = var.web_host
    }
  }

@@ -97,7 +97,7 @@ resource "google_monitoring_uptime_check_config" "web-https" {
 }

 resource "google_monitoring_alert_policy" "instances_high_cpu_policy" {
-  project = module.google-cloud-project.project.project_id
+  project = var.project_id

  display_name = "High Instance CPU utilization"
  combiner     = "OR"
@@ -134,7 +134,7 @@ resource "google_monitoring_alert_policy" "instances_high_cpu_policy" {
 }

 resource "google_monitoring_alert_policy" "sql_high_cpu_policy" {
-  project = module.google-cloud-project.project.project_id
+  project = var.project_id

  display_name = "High Cloud SQL CPU utilization"
  combiner     = "OR"
@@ -171,7 +171,7 @@ resource "google_monitoring_alert_policy" "sql_high_cpu_policy" {
 }

 resource "google_monitoring_alert_policy" "sql_disk_utiliziation_policy" {
-  project = module.google-cloud-project.project.project_id
+  project = var.project_id

  display_name = "High Cloud SQL Disk utilization"
  combiner     = "OR"
--- a/terraform/modules/google-cloud-ops/variables.tf
+++ b/terraform/modules/google-cloud-ops/variables.tf
@@ -0,0 +1,21 @@
+variable "project_id" {
+  description = "The ID of the project in which the resource belongs."
+}
+
+variable "slack_alerts_channel" {
+  type        = string
+  description = "Slack channel which will receive monitoring alerts"
+}
+
+variable "slack_alerts_auth_token" {
+  type        = string
+  description = "Slack auth token for the infra alerts channel"
+}
+
+variable "api_host" {
+  type = string
+}
+
+variable "web_host" {
+  type = string
+}
--- a/terraform/modules/google-cloud-vpc/main.tf
+++ b/terraform/modules/google-cloud-vpc/main.tf
@@ -17,3 +17,33 @@ resource "google_compute_network" "vpc_network" {
    google_project_service.compute
  ]
 }
+
+## Router and Cloud NAT are required for instances without external IP address
+resource "google_compute_router" "default" {
+  project = var.project_id
+
+  name    = google_compute_network.vpc_network.name
+  network = google_compute_network.vpc_network.self_link
+  region  = var.nat_region
+}
+
+resource "google_compute_router_nat" "application" {
+  project = var.project_id
+
+  name   = google_compute_network.vpc_network.name
+  region = var.nat_region
+
+  router = google_compute_router.default.name
+
+  nat_ip_allocate_option             = "AUTO_ONLY"
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+
+  enable_dynamic_port_allocation = false
+  min_ports_per_vm               = 32
+
+  udp_idle_timeout_sec             = 30
+  icmp_idle_timeout_sec            = 30
+  tcp_established_idle_timeout_sec = 1200
+  tcp_transitory_idle_timeout_sec  = 30
+  tcp_time_wait_timeout_sec        = 120
+}
--- a/terraform/modules/google-cloud-vpc/variables.tf
+++ b/terraform/modules/google-cloud-vpc/variables.tf
@@ -5,3 +5,7 @@ variable "project_id" {
 variable "name" {
  description = "Name of the resource. Provided by the client when the resource is created."
 }
+
+variable "nat_region" {
+  description = "Region where Cloud NAT will be created"
+}
--- a/terraform/modules/relay-app/main.tf
+++ b/terraform/modules/relay-app/main.tf
@@ -5,7 +5,6 @@ locals {
  application_labels = merge({
    managed_by  = "terraform"
    application = local.application_name
-    version     = local.application_version
  }, var.application_labels)

  google_health_check_ip_ranges = [
@@ -143,7 +142,7 @@ resource "google_compute_subnetwork" "subnetwork" {
  network = google_compute_network.network.self_link

  stack_type               = "IPV4_IPV6"
-  ip_cidr_range            = "10.128.0.0/20"
+  ip_cidr_range            = "10.${129 + index(keys(var.instances), each.key)}.0.0/24"
  ipv6_access_type         = "EXTERNAL"
  private_ip_google_access = true
 }
@@ -166,6 +165,7 @@ resource "google_compute_instance_template" "application" {

  labels = merge({
    container-vm = data.google_compute_image.coreos.name
+    version      = local.application_version
  }, local.application_labels)

  scheduling {