feat(portal): Add Oban (#8786)

Our current bespoke job system, while it's worked out well so far, has
the following shortcomings:

- No retry logic
- No robust to guarantee job isolation / uniqueness without resorting to
row-level locking
- No support for cron-based scheduling

This PR adds the boilerplate required to get started with
[Oban](https://hexdocs.pm/oban/Oban.html), the job management system for
Elixir.

elixir/apps/domain/lib/domain/application.ex

@@ -4,6 +4,9 @@ defmodule Domain.Application do
   def start(_type, _args) do
     configure_logger()
 
+    # Attach Oban Sentry reporter
+    Domain.Telemetry.Reporter.Oban.attach()
+
     _ = OpentelemetryLoggerMetadata.setup()
     _ = OpentelemetryEcto.setup([:domain, :repo])
 
@@ -23,6 +26,7 @@ defmodule Domain.Application do
       # Note: only one of platform adapters will be actually started.
       Domain.GoogleCloudPlatform,
       Domain.Cluster,
+      {Oban, Application.fetch_env!(:domain, Oban)},
 
       # Application
       Domain.Tokens,
@@ -42,6 +46,9 @@ defmodule Domain.Application do
   end
 
   defp configure_logger do
+    # Attach Oban to the logger
+    Oban.Telemetry.attach_default_logger(encode: false, level: log_level())
+
     # Configure Logger severity at runtime
     :ok = LoggerJSON.configure_log_level_from_env!("LOG_LEVEL")
 
@@ -59,4 +66,13 @@ defmodule Domain.Application do
       }
     })
   end
+
+  defp log_level do
+    case System.get_env("LOG_LEVEL") do
+      "error" -> :error
+      "warn" -> :warn
+      "debug" -> :debug
+      _ -> :info
+    end
+  end
 end

elixir/apps/domain/lib/domain/telemetry/reporter/oban.ex

@@ -0,0 +1,18 @@
+defmodule Domain.Telemetry.Reporter.Oban do
+  @moduledoc """
+  A simple module for reporting Oban job exceptions to Sentry.
+  """
+
+  def attach do
+    :telemetry.attach("oban-errors", [:oban, :job, :exception], &__MODULE__.handle_event/4, [])
+  end
+
+  def handle_event([:oban, :job, :exception], measure, meta, _) do
+    extra =
+      meta.job
+      |> Map.take([:id, :args, :meta, :queue, :worker])
+      |> Map.merge(measure)
+
+    Sentry.capture_exception(meta.reason, stacktrace: meta.stacktrace, extra: extra)
+  end
+end

elixir/apps/domain/mix.exs

@@ -59,6 +59,9 @@ defmodule Domain.MixProject do
       {:argon2_elixir, "~> 4.0"},
       {:workos, "~> 1.1"},
 
+      # Job system
+      {:oban, "~> 2.19"},
+
       # Erlang Clustering
       {:libcluster, "~> 3.3"},
 

elixir/apps/domain/priv/repo/migrations/20250415020405_add_oban_jobs_table.exs

@@ -0,0 +1,13 @@
+defmodule Domain.Repo.Migrations.AddObanJobsTable do
+  use Ecto.Migration
+
+  def up do
+    Oban.Migration.up(version: 12)
+  end
+
+  # We specify `version: 1` in `down`, ensuring that we'll roll all the way back down if
+  # necessary, regardless of which version we've migrated `up` to.
+  def down do
+    Oban.Migration.down(version: 1)
+  end
+end

elixir/config/config.exs

@@ -113,6 +113,11 @@ config :domain, outbound_email_adapter_configured?: false
 
 config :domain, web_external_url: "http://localhost:13000"
 
+config :domain, Oban,
+  engine: Oban.Engines.Basic,
+  queues: [default: 10],
+  repo: Domain.Repo
+
 ###############################
 ##### Web #####################
 ###############################

elixir/config/prod.exs

@@ -8,6 +8,18 @@ config :domain, Domain.Repo,
   pool_size: 10,
   show_sensitive_data_on_connection_error: false
 
+config :domain, Oban,
+  plugins: [
+    # Keep the last 90 days of completed, cancelled, and discarded jobs
+    {Oban.Plugins.Pruner, max_age: 60 * 60 * 24 * 90}
+
+    # Rescue jobs that may have failed due to transient errors like deploys
+    # or network issues. It's not guaranteed that the job won't be executed
+    # twice, so for now we disable it since all of our Oban jobs can be retried
+    # without loss.
+    # {Oban.Plugins.Lifeline, rescue_after: :timer.minutes(30)}
+  ]
+
 ###############################
 ##### Web #####################
 ###############################

elixir/config/test.exs

@@ -40,6 +40,9 @@ config :domain, Domain.Telemetry.Reporter.GoogleCloudMetrics, project_id: "fz-te
 
 config :domain, web_external_url: "http://localhost:13100"
 
+# Prevent Oban from running jobs and plugins in tests
+config :domain, Oban, testing: :manual
+
 ###############################
 ##### Web #####################
 ###############################

elixir/mix.lock

@@ -62,6 +62,7 @@
   "nimble_ownership": {:hex, :nimble_ownership, "1.0.1", "f69fae0cdd451b1614364013544e66e4f5d25f36a2056a9698b793305c5aa3a6", [:mix], [], "hexpm", "3825e461025464f519f3f3e4a1f9b68c47dc151369611629ad08b636b73bb22d"},
   "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
   "number": {:hex, :number, "1.0.5", "d92136f9b9382aeb50145782f116112078b3465b7be58df1f85952b8bb399b0f", [:mix], [{:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "c0733a0a90773a66582b9e92a3f01290987f395c972cb7d685f51dd927cd5169"},
+  "oban": {:hex, :oban, "2.19.4", "045adb10db1161dceb75c254782f97cdc6596e7044af456a59decb6d06da73c1", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5fcc6219e6464525b808d97add17896e724131f498444a292071bf8991c99f97"},
   "observer_cli": {:hex, :observer_cli, "1.8.2", "9962e6818e75774ec829875016be61a6bca87e95ad18ecd7fbcf4f23f1278c57", [:mix, :rebar3], [{:recon, "~> 2.5.6", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "93ae523d42d566b176f7ae77a0bf36802dab8bb51a6086316cce66a7cfb5d81f"},
   "open_api_spex": {:hex, :open_api_spex, "3.21.2", "6a704f3777761feeb5657340250d6d7332c545755116ca98f33d4b875777e1e5", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "f42ae6ed668b895ebba3e02773cfb4b41050df26f803f2ef634c72a7687dc387"},
   "openid_connect": {:git, "https://github.com/firezone/openid_connect.git", "e4d9dca8ae43c765c00a7d3dfa12d6f24f5b3418", [ref: "e4d9dca8ae43c765c00a7d3dfa12d6f24f5b3418"]},