feat(portal): Add IDP sync error email notifications (#6483)

This adds a feature that will email all admins in a Firezone Account
when sync errors occur with their Identity Provider.

In order to avoid spamming admins with sync error emails, the error
emails are only sent once every 24 hours. One exception to that is when
there is a successful sync the `sync_error_emailed_at` field is reset,
which means in theory if an identity provider was flip flopping between
successful and unsuccessful syncs the admins would be emailed more than
once in a 24 hours period.

### Sample Email Message
<img width="589" alt="idp-sync-error-message"
src="https://github.com/user-attachments/assets/d7128c7c-c10d-4d02-8283-059e2f1f5db5">

elixir/README.md

@@ -418,7 +418,7 @@ iex(web@web-3vmw.us-east1-d.c.firezone-staging.internal)5> context = %Domain.Aut
 iex(web@web-3vmw.us-east1-d.c.firezone-staging.internal)6> {:ok, identity} = Domain.Auth.Adapters.Email.request_sign_in_token(identity, context)
 {:ok, ...}
 
-iex(web@web-3vmw.us-east1-d.c.firezone-staging.internal)7> Web.Mailer.AuthEmail.sign_in_link_email(identity) |> Web.Mailer.deliver()
+iex(web@web-3vmw.us-east1-d.c.firezone-staging.internal)7> Domain.Mailer.AuthEmail.sign_in_link_email(identity) |> Domain.Mailer.deliver()
 {:ok, %{id: "d24dbe9a-d0f5-4049-ac0d-0df793725a80"}}
 ```
 

elixir/apps/api/lib/api/plugs/auth.ex

@@ -11,12 +11,6 @@ defmodule API.Plugs.Auth do
       assign(conn, :subject, subject)
     else
       _ ->
-        # conn
-        # |> put_resp_content_type("application/json")
-        # |> send_resp(401, Jason.encode!(%{"error" => "invalid_access_token"}))
-        # |> halt()
-
-        # TODO: BRIAN - Confirm that this change won't break anything with the clients or gateways
         conn
         |> put_status(401)
         |> Phoenix.Controller.put_view(json: API.ErrorJSON)

elixir/apps/domain/.formatter.exs

@@ -1,7 +1,8 @@
 [
   import_deps: [
     :ecto,
-    :plug
+    :plug,
+    :phoenix
   ],
   inputs: [
     "*.{heex,ex,exs}",

elixir/apps/domain/lib/domain/actors.ex

@@ -402,6 +402,16 @@ defmodule Domain.Actors do
     end)
   end
 
+  def all_admins_for_account!(%Accounts.Account{} = account, opts \\ []) do
+    {preload, _opts} = Keyword.pop(opts, :preload, [])
+
+    Actor.Query.not_disabled()
+    |> Actor.Query.by_account_id(account.id)
+    |> Actor.Query.by_type(:account_admin_user)
+    |> Repo.all(opts)
+    |> Repo.preload(preload)
+  end
+
   def list_actors(%Auth.Subject{} = subject, opts \\ []) do
     with :ok <- Auth.ensure_has_permissions(subject, Authorizer.manage_actors_permission()) do
       Actor.Query.not_deleted()

elixir/apps/domain/lib/domain/application.ex

@@ -30,6 +30,8 @@ defmodule Domain.Application do
       Domain.Gateways,
       Domain.Clients,
       Domain.Billing,
+      Domain.Mailer,
+      Domain.Mailer.RateLimiter,
 
       # Observability
       Domain.Telemetry

elixir/apps/domain/lib/domain/auth.ex

@@ -352,6 +352,12 @@ defmodule Domain.Auth do
     end
   end
 
+  def all_identities_for(%Actors.Actor{} = actor, opts \\ []) do
+    Identity.Query.not_deleted()
+    |> Identity.Query.by_actor_id(actor.id)
+    |> Repo.all(opts)
+  end
+
   def list_identities_for(%Actors.Actor{} = actor, %Subject{} = subject, opts \\ []) do
     with :ok <- ensure_has_permissions(subject, Authorizer.manage_identities_permission()) do
       Identity.Query.not_deleted()
@@ -373,6 +379,19 @@ defmodule Domain.Auth do
     |> Repo.all()
   end
 
+  def get_identity_email(%Identity{} = identity) do
+    provider_email(identity) || identity.provider_identifier
+  end
+
+  def identity_has_email?(%Identity{} = identity) do
+    not is_nil(provider_email(identity)) or identity.provider.adapter == :email or
+      identity.provider_identifier =~ "@"
+  end
+
+  defp provider_email(%Identity{} = identity) do
+    get_in(identity.provider_state, ["userinfo", "email"])
+  end
+
   # used by IdP adapters
   def upsert_identity(%Actors.Actor{} = actor, %Provider{} = provider, attrs) do
     Identity.Changeset.create_identity(actor, provider, attrs)

elixir/apps/domain/lib/domain/auth/adapter/openid_connect/directory_sync.ex

@@ -209,6 +209,7 @@ defmodule Domain.Auth.Adapter.OpenIDConnect.DirectorySync do
 
           Auth.Provider.Changeset.sync_requires_manual_intervention(provider, user_message)
           |> Domain.Repo.update!()
+          |> send_sync_error_email()
 
           :error
 
@@ -224,6 +225,7 @@ defmodule Domain.Auth.Adapter.OpenIDConnect.DirectorySync do
 
           Auth.Provider.Changeset.sync_failed(provider, user_message)
           |> Domain.Repo.update!()
+          |> send_sync_error_email()
           |> log_sync_error(log_message)
 
           :error
@@ -382,6 +384,37 @@ defmodule Domain.Auth.Adapter.OpenIDConnect.DirectorySync do
     end)
   end
 
+  defp send_sync_error_email(provider) do
+    provider = Repo.preload(provider, :account)
+
+    if sync_error_email_sent_today?(provider) do
+      Logger.debug("Sync error email already sent today")
+
+      provider
+    else
+      Domain.Actors.all_admins_for_account!(provider.account, preload: :identities)
+      |> Enum.flat_map(fn actor ->
+        Enum.map(actor.identities, &Domain.Auth.get_identity_email(&1))
+      end)
+      |> Enum.uniq()
+      |> Enum.each(fn email ->
+        Domain.Mailer.SyncEmail.sync_error_email(provider, email)
+        |> Domain.Mailer.deliver()
+      end)
+
+      Auth.Provider.Changeset.sync_error_emailed(provider)
+      |> Domain.Repo.update!()
+    end
+  end
+
+  defp sync_error_email_sent_today?(provider) do
+    if last_email_time = provider.sync_error_emailed_at do
+      DateTime.diff(DateTime.utc_now(), last_email_time, :hour) < 24
+    else
+      false
+    end
+  end
+
   if Mix.env() == :test do
     # We need this function to reuse the connection that was checked out in a parent process.
     #

elixir/apps/domain/lib/domain/auth/adapters/okta/api_client.ex

@@ -100,6 +100,7 @@ defmodule Domain.Auth.Adapters.Okta.APIClient do
     end
   end
 
+  # TODO: Need to catch 401/403 specifically when error message is in header
   defp list(uri, headers, api_token) do
     headers = headers ++ [{"Authorization", "Bearer #{api_token}"}]
     request = Finch.build(:get, uri, headers)

elixir/apps/domain/lib/domain/auth/adapters/okta/jobs/sync_directory.ex

@@ -46,6 +46,16 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectory do
         message = "#{error_code} => #{error_summary}"
         {:error, message, "Okta API returned #{status}: #{message}"}
 
+      # TODO: Okta API client needs to be updated to pull message from header
+      {:error, {401, ""}} ->
+        message = "401 - Unauthorized"
+        {:error, message, message}
+
+      # TODO: Okta API client needs to be updated to pull message from header
+      {:error, {403, ""}} ->
+        message = "403 - Forbidden"
+        {:error, message, message}
+
       {:error, :retry_later} ->
         message = "Okta API is temporarily unavailable"
         {:error, message, message}

elixir/apps/domain/lib/domain/auth/provider.ex

@@ -23,6 +23,7 @@ defmodule Domain.Auth.Provider do
     field :last_sync_error, :string
     field :last_synced_at, :utc_datetime_usec
     field :sync_disabled_at, :utc_datetime_usec
+    field :sync_error_emailed_at, :utc_datetime_usec
 
     field :disabled_at, :utc_datetime_usec
     field :deleted_at, :utc_datetime_usec

elixir/apps/domain/lib/domain/auth/provider/changeset.ex

@@ -5,7 +5,7 @@ defmodule Domain.Auth.Provider.Changeset do
 
   @create_fields ~w[id name adapter provisioner adapter_config adapter_state disabled_at]a
   @update_fields ~w[name adapter_config
-                    last_syncs_failed last_sync_error sync_disabled_at
+                    last_syncs_failed last_sync_error sync_disabled_at sync_error_emailed_at
                     adapter_state provisioner disabled_at deleted_at]a
   @required_fields ~w[name adapter adapter_config provisioner]a
 
@@ -47,9 +47,9 @@ defmodule Domain.Auth.Provider.Changeset do
     provider
     |> change()
     |> put_change(:last_synced_at, DateTime.utc_now())
-    |> put_change(:last_sync_error, nil)
     |> put_change(:last_syncs_failed, 0)
     |> put_change(:sync_disabled_at, nil)
+    |> put_change(:sync_error_emailed_at, nil)
   end
 
   def sync_failed(%Provider{} = provider, error) do
@@ -62,6 +62,12 @@ defmodule Domain.Auth.Provider.Changeset do
     |> put_change(:last_syncs_failed, last_syncs_failed + 1)
   end
 
+  def sync_error_emailed(%Provider{} = provider) do
+    provider
+    |> change()
+    |> put_change(:sync_error_emailed_at, DateTime.utc_now())
+  end
+
   def sync_requires_manual_intervention(%Provider{} = provider, error) do
     sync_failed(provider, error)
     |> put_change(:sync_disabled_at, DateTime.utc_now())

elixir/apps/domain/lib/domain/cldr.ex

@@ -0,0 +1,5 @@
+defmodule Domain.CLDR do
+  use Cldr,
+    locales: ["en"],
+    providers: [Cldr.Number, Cldr.Calendar, Cldr.DateTime]
+end

elixir/apps/domain/lib/domain/mailer.ex

@@ -1,8 +1,8 @@
-defmodule Web.Mailer do
+defmodule Domain.Mailer do
   use Supervisor
   alias Swoosh.Mailer
   alias Swoosh.Email
-  alias Web.Mailer.RateLimiter
+  alias Domain.Mailer.RateLimiter
   require Logger
 
   def start_link(arg) do
@@ -42,7 +42,7 @@ defmodule Web.Mailer do
   custom adapter implementation that does nothing.
   """
   def deliver(email, config \\ []) do
-    opts = Mailer.parse_config(:web, __MODULE__, [], config)
+    opts = Mailer.parse_config(:domain, __MODULE__, [], config)
     metadata = %{email: email, config: config, mailer: __MODULE__}
 
     if opts[:adapter] do
@@ -80,14 +80,14 @@ defmodule Web.Mailer do
   end
 
   def active? do
-    mailer_config = Domain.Config.fetch_env!(:web, Web.Mailer)
+    mailer_config = Domain.Config.fetch_env!(:domain, Domain.Mailer)
     mailer_config[:from_email] && mailer_config[:adapter]
   end
 
   def default_email do
     # Fail hard if email not configured
     from_email =
-      Domain.Config.fetch_env!(:web, Web.Mailer)
+      Domain.Config.fetch_env!(:domain, Domain.Mailer)
       |> Keyword.fetch!(:from_email)
 
     Email.new()

elixir/apps/domain/lib/domain/mailer/auth_email.ex

@@ -1,7 +1,7 @@
-defmodule Web.Mailer.AuthEmail do
-  use Web, :html
+defmodule Domain.Mailer.AuthEmail do
   import Swoosh.Email
-  import Web.Mailer
+  import Domain.Mailer
+  import Phoenix.Template, only: [embed_templates: 2]
 
   embed_templates "auth_email/*.html", suffix: "_html"
   embed_templates "auth_email/*.text", suffix: "_text"
@@ -12,7 +12,7 @@ defmodule Web.Mailer.AuthEmail do
         user_agent,
         remote_ip
       ) do
-    sign_in_form_url = url(~p"/#{account}")
+    sign_in_form_url = url("/#{account.slug}")
 
     default_email()
     |> subject("Welcome to Firezone")
@@ -40,11 +40,12 @@ defmodule Web.Mailer.AuthEmail do
 
     sign_in_url =
       url(
-        ~p"/#{identity.account}/sign_in/providers/#{identity.provider_id}/verify_sign_in_token?#{params}"
+        "/#{identity.account.slug}/sign_in/providers/#{identity.provider_id}/verify_sign_in_token",
+        params
       )
 
     sign_in_token_created_at =
-      Cldr.DateTime.to_string!(identity.provider_state["token_created_at"], Web.CLDR,
+      Cldr.DateTime.to_string!(identity.provider_state["token_created_at"], Domain.CLDR,
         format: :short
       ) <> " UTC"
 
@@ -69,11 +70,27 @@ defmodule Web.Mailer.AuthEmail do
       ) do
     default_email()
     |> subject("Welcome to Firezone")
-    |> to(get_identity_email(identity))
+    |> to(Domain.Auth.get_identity_email(identity))
     |> render_body(__MODULE__, :new_user,
       account: account,
       identity: identity,
       subject: subject
     )
   end
+
+  def url(path, params \\ %{}) do
+    Domain.Config.fetch_env!(:domain, :web_external_url)
+    |> URI.parse()
+    |> URI.append_path(path)
+    |> maybe_append_query(params)
+    |> URI.to_string()
+  end
+
+  def maybe_append_query(uri, params) do
+    if Enum.empty?(params) do
+      uri
+    else
+      URI.append_query(uri, URI.encode_query(params))
+    end
+  end
 end

elixir/apps/domain/lib/domain/mailer/beta_email.ex

@@ -1,7 +1,7 @@
-defmodule Web.Mailer.BetaEmail do
-  use Web, :html
+defmodule Domain.Mailer.BetaEmail do
   import Swoosh.Email
-  import Web.Mailer
+  import Domain.Mailer
+  import Phoenix.Template, only: [embed_templates: 2]
 
   embed_templates "beta_email/*.text", suffix: "_text"
 

elixir/apps/domain/lib/domain/mailer/rate_limiter.ex

@@ -1,4 +1,4 @@
-defmodule Web.Mailer.RateLimiter do
+defmodule Domain.Mailer.RateLimiter do
   use GenServer
 
   @default_ets_table_name __MODULE__.ETS

elixir/apps/domain/lib/domain/mailer/sync_email.ex

@@ -0,0 +1,15 @@
+defmodule Domain.Mailer.SyncEmail do
+  import Swoosh.Email
+  import Domain.Mailer
+  import Phoenix.Template, only: [embed_templates: 2]
+
+  embed_templates "sync_email/*.html", suffix: "_html"
+  embed_templates "sync_email/*.text", suffix: "_text"
+
+  def sync_error_email(%Domain.Auth.Provider{} = provider, email) do
+    default_email()
+    |> subject("Firezone Identity Provider Sync Error")
+    |> to(email)
+    |> render_body(__MODULE__, :sync_error, account: provider.account, provider: provider)
+  end
+end

elixir/apps/domain/lib/domain/mailer/sync_email/sync_error.html.eex

@@ -0,0 +1,136 @@
+<!DOCTYPE html>
+<html lang="en" xmlns:v="urn:schemas-microsoft-com:vml">
+	<head>
+		<meta charset="utf-8">
+		<meta name="x-apple-disable-message-reformatting">
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+		<meta name="format-detection" content="telephone=no, date=no, address=no, email=no, url=no">
+		<meta name="color-scheme" content="light dark">
+		<meta name="supported-color-schemes" content="light dark">
+		<!--[if mso]>
+		<noscript>
+			<xml>
+				<o:OfficeDocumentSettings xmlns:o="urn:schemas-microsoft-com:office:office">
+					<o:PixelsPerInch>96</o:PixelsPerInch>
+				</o:OfficeDocumentSettings>
+			</xml>
+		</noscript>
+		<style>
+			td,th,div,p,a,h1,h2,h3,h4,h5,h6 {font-family: "Segoe UI", sans-serif; mso-line-height-rule: exactly;}
+		</style>
+		<![endif]-->
+		<title>Firezone Sync Error</title>
+		<style>
+			.hover-important-text-decoration-underline:hover {
+				text-decoration: underline !important
+			}
+			:is([dir="rtl"] .rtl-text-right) {
+				text-align: right !important
+			}
+			@media (max-width: 600px) {
+				.sm-my-8 {
+					margin-top: 32px !important;
+					margin-bottom: 32px !important
+				}
+				.sm-px-4 {
+					padding-left: 16px !important;
+					padding-right: 16px !important
+				}
+				.sm-px-6 {
+					padding-left: 24px !important;
+					padding-right: 24px !important
+				}
+				.sm-leading-8 {
+					line-height: 32px !important
+				}
+			}
+		</style>
+	</head>
+	<body style="margin: 0; width: 100%; background-color: #f6f6f6; padding: 0; -webkit-font-smoothing: antialiased; word-break: break-word">
+		<div role="article" aria-roledescription="email" aria-label="Firezone Sync Error" lang="en">
+			<div class="sm-px-4" style="background-color: #f6f6f6; font-family: ui-sans-serif, system-ui, -apple-system, 'Segoe UI', sans-serif">
+				<table align="center" cellpadding="0" cellspacing="0" role="none">
+					<tr>
+						<td style="width: 552px; max-width: 100%">
+							<div class="sm-my-8" style="margin-top: 48px; margin-bottom: 48px; text-align: center">
+								<a href="https://firezone.dev">
+									<div>
+										<img src="https://www.firezone.dev/images/logo-lockup.png" width="250" alt="Firezone logo" style="max-width: 100%; vertical-align: middle; line-height: 1">
+									</div>
+								</a>
+							</div>
+							<table style="width: 100%;" cellpadding="0" cellspacing="0" role="none">
+								<tr>
+									<td class="sm-px-6" style="border-radius: 4px; background-color: #ffffff; padding: 48px; font-size: 16px; color: #4f4f4f; box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05)">
+										<h1 class="sm-leading-8" style="margin: 0 0 24px; font-size: 24px; font-weight: 600; color: #000000">
+											<%= @provider.name %> Sync Error!
+										</h1>
+										<p style="margin: 0; line-height: 24px">
+											<%= @provider.name %> has failed to sync <%= @provider.last_syncs_failed %> times.
+										</p>
+										<div role="separator" style="line-height: 16px">&zwj;</div>
+										<p style="margin: 0; line-height: 24px;">
+											Below is the last sync error message:
+											<div>
+											<pre style="margin: 0; white-space: pre; border-radius: 4px; background-color: #000000; padding: 8px 12px; line-height: 24px; color: #e7e7e7"><code><%= @provider.last_sync_error %></code></pre>
+											</div>
+										</p>
+										<div role="separator" style="background-color: #d1d1d1; height: 1px; line-height: 1px; margin: 32px 0">&zwj;</div>
+										<p style="margin: 0; line-height: 24px;">
+											<span style="font-weight: 500; text-decoration-line: underline; text-underline-offset: 2px">Identity Provider Details</span>
+										</p>
+										<table class="rtl-text-right" style="width: 100%; text-align: left; font-size: 14px; color: #6d6d6d" cellpadding="0" cellspacing="0" role="none">
+											<tr style="border-bottom-width: 1px; background-color: #ffffff">
+												<th scope="row" style="white-space: nowrap; padding-left: 24px; padding-right: 24px; font-weight: 500; color: #3d3d3d">Account</th>
+                        <td style="padding: 4px 24px"><%= @account.name %></td>
+											</tr>
+											<tr style="border-bottom-width: 1px; background-color: #ffffff;">
+												<th scope="row" style="white-space: nowrap; padding-left: 24px; padding-right: 24px; font-weight: 500; color: #3d3d3d;">Account ID</th>
+												<td style="padding: 4px 24px;"><%= @account.id %></td>
+											</tr>
+											<tr style="border-bottom-width: 1px; background-color: #ffffff;">
+												<th scope="row" style="white-space: nowrap; padding-left: 24px; padding-right: 24px; font-weight: 500; color: #3d3d3d;">Provider Name</th>
+												<td style="padding: 4px 24px;"><%= @provider.name %></td>
+											</tr>
+											<tr style="border-bottom-width: 1px; background-color: #ffffff;">
+												<th scope="row" style="white-space: nowrap; padding-left: 24px; padding-right: 24px; font-weight: 500; color: #3d3d3d;">Provider ID</th>
+												<td style="padding: 4px 24px;"><%= @provider.id %></td>
+											</tr>
+										</table>
+										<p></p>
+										<div role="separator" style="background-color: #d1d1d1; height: 1px; line-height: 1px; margin: 32px 0;">&zwj;</div>
+										<p style="margin: 0; line-height: 24px;">
+											Please verify that all Identity Provider information entered in to Firezone is correct. If the problem persists, please reach out to Firezone support
+											using Slack or email.
+											<br>
+											<br>
+											Thanks, <br>The Firezone Team
+										</p>
+										<div role="separator" style="line-height: 16px">&zwj;</div>
+									</td>
+								</tr>
+								<tr role="separator">
+									<td style="line-height: 48px">&zwj;</td>
+								</tr>
+								<tr>
+									<td style="padding-left: 24px; padding-right: 24px; text-align: center; font-size: 12px; color: #575757">
+										<p style="margin: 0; font-style: italic">
+											Blazing-fast alternative to legacy VPNs
+										</p>
+										<p style="cursor: default">
+											<a href="https://www.firezone.dev/kb" class="hover-important-text-decoration-underline" style="color: #37007f; text-decoration: none">Docs</a>
+											&bull;
+											<a href="https://github.com/firezone" class="hover-important-text-decoration-underline" style="color: #37007f; text-decoration: none;">Github</a>
+											&bull;
+											<a href="https://x.com/firezonehq" class="hover-important-text-decoration-underline" style="color: #37007f; text-decoration: none;">X</a>
+										</p>
+									</td>
+								</tr>
+							</table>
+						</td>
+					</tr>
+				</table>
+			</div>
+		</div>
+	</body>
+</html>

elixir/apps/domain/lib/domain/mailer/sync_email/sync_error.text.eex

@@ -0,0 +1,12 @@
+Identity Provider Sync Error!
+
+An Identity Provider in your Firezone Account has failed to sync <%= @provider.last_syncs_failed%> times.
+
+The following is the last sync error message:
+<%= @provider.last_sync_error %>
+
+Identity Provider details:
+  Account: <%= @account.name %>
+  Account ID: <%= @account.id %>
+  Provider Name: <%= @provider.name %>
+  Provider ID: <%= @provider.id %>

elixir/apps/domain/mix.exs

@@ -64,6 +64,18 @@ defmodule Domain.MixProject do
       # Erlang Clustering
       {:libcluster, "~> 3.3"},
 
+      # CLDR and unit conversions
+      {:ex_cldr_dates_times, "~> 2.13"},
+      {:ex_cldr_numbers, "~> 2.31"},
+      {:ex_cldr, "~> 2.38"},
+
+      # Mailer deps
+      {:gen_smtp, "~> 1.0"},
+      {:multipart, "~> 0.4.0"},
+      {:phoenix_html, "~> 4.0"},
+      {:phoenix_swoosh, "~> 1.0"},
+      {:phoenix_template, "~> 1.0.4"},
+
       # Observability and Runtime debugging
       {:bandit, "~> 1.0"},
       {:plug, "~> 1.15"},

elixir/apps/domain/priv/repo/migrations/20240823195642_add_sync_error_email_to_provider.exs

@@ -0,0 +1,9 @@
+defmodule Domain.Repo.Migrations.AddSyncErrorEmailToProvider do
+  use Ecto.Migration
+
+  def change do
+    alter table(:auth_providers) do
+      add(:sync_error_emailed_at, :utc_datetime_usec)
+    end
+  end
+end

elixir/apps/domain/test/domain/auth/adapters/google_workspace/jobs/sync_directory_test.exs

@@ -774,5 +774,75 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.Jobs.SyncDirectoryTest do
 
       cancel_bypass_expectations_check(bypass)
     end
+
+    test "sends email on failed directory sync", %{account: account} do
+      actor = Fixtures.Actors.create_actor(type: :account_admin_user, account: account)
+      _identity = Fixtures.Auth.create_identity(account: account, actor: actor)
+
+      error_message =
+        "Admin SDK API has not been used in project XXXX before or it is disabled. " <>
+          "Enable it by visiting https://console.developers.google.com/apis/api/admin.googleapis.com/overview?project=XXXX " <>
+          "then retry. If you enabled this API recently, wait a few minutes for the action to propagate to our systems and retry."
+
+      response = %{
+        "error" => %{
+          "code" => 403,
+          "message" => error_message,
+          "errors" => [
+            %{
+              "message" => error_message,
+              "domain" => "usageLimits",
+              "reason" => "accessNotConfigured",
+              "extendedHelp" => "https://console.developers.google.com"
+            }
+          ],
+          "status" => "PERMISSION_DENIED",
+          "details" => [
+            %{
+              "@type" => "type.googleapis.com/google.rpc.Help",
+              "links" => [
+                %{
+                  "description" => "Google developers console API activation",
+                  "url" =>
+                    "https://console.developers.google.com/apis/api/admin.googleapis.com/overview?project=100421656358"
+                }
+              ]
+            },
+            %{
+              "@type" => "type.googleapis.com/google.rpc.ErrorInfo",
+              "reason" => "SERVICE_DISABLED",
+              "domain" => "googleapis.com",
+              "metadata" => %{
+                "service" => "admin.googleapis.com",
+                "consumer" => "projects/100421656358"
+              }
+            }
+          ]
+        }
+      }
+
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.override_endpoint_url("http://localhost:#{bypass.port}/")
+
+      for path <- [
+            "/admin/directory/v1/users",
+            "/admin/directory/v1/customer/my_customer/orgunits",
+            "/admin/directory/v1/groups"
+          ] do
+        Bypass.stub(bypass, "GET", path, fn conn ->
+          Plug.Conn.send_resp(conn, 403, Jason.encode!(response))
+        end)
+      end
+
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
+
+      assert_email_sent(fn email ->
+        assert email.subject == "Firezone Identity Provider Sync Error"
+        assert email.text_body =~ "failed to sync 1 times"
+      end)
+
+      cancel_bypass_expectations_check(bypass)
+    end
   end
 end

elixir/apps/domain/test/domain/auth/adapters/jumpcloud/jobs/sync_directory_test.exs

@@ -561,5 +561,36 @@ defmodule Domain.Auth.Adapters.JumpCloud.Jobs.SyncDirectoryTest do
 
       cancel_bypass_expectations_check(bypass)
     end
+
+    test "sends email on failed directory sync", %{account: account} do
+      actor = Fixtures.Actors.create_actor(type: :account_admin_user, account: account)
+      _identity = Fixtures.Auth.create_identity(account: account, actor: actor)
+
+      bypass = Bypass.open()
+
+      WorkOSDirectory.override_base_url("http://localhost:#{bypass.port}")
+
+      for path <- [
+            "/directories",
+            "/directory_users",
+            "/directory_groups"
+          ] do
+        Bypass.stub(bypass, "GET", path, fn conn ->
+          conn
+          |> Plug.Conn.prepend_resp_headers([{"content-type", "application/json"}])
+          |> Plug.Conn.send_resp(500, Jason.encode!(%{}))
+        end)
+      end
+
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
+
+      assert_email_sent(fn email ->
+        assert email.subject == "Firezone Identity Provider Sync Error"
+        assert email.text_body =~ "failed to sync 1 times"
+      end)
+
+      cancel_bypass_expectations_check(bypass)
+    end
   end
 end

elixir/apps/domain/test/domain/auth/adapters/microsoft_entra/jobs/sync_directory_test.exs

@@ -483,5 +483,32 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.Jobs.SyncDirectoryTest do
 
       cancel_bypass_expectations_check(bypass)
     end
+
+    test "sends email on failed directory sync", %{account: account} do
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.override_endpoint_url("http://localhost:#{bypass.port}/")
+
+      actor = Fixtures.Actors.create_actor(type: :account_admin_user, account: account)
+      _identity = Fixtures.Auth.create_identity(account: account, actor: actor)
+
+      for path <- [
+            "v1.0/users",
+            "v1.0/groups"
+          ] do
+        Bypass.stub(bypass, "GET", path, fn conn ->
+          Plug.Conn.send_resp(conn, 500, "")
+        end)
+      end
+
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
+
+      assert_email_sent(fn email ->
+        assert email.subject == "Firezone Identity Provider Sync Error"
+        assert email.text_body =~ "failed to sync 1 times"
+      end)
+
+      cancel_bypass_expectations_check(bypass)
+    end
   end
 end

elixir/apps/domain/test/domain/auth/adapters/okta/jobs/sync_directory_test.exs

@@ -33,8 +33,6 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
     end
 
     test "syncs IdP data", %{provider: provider, bypass: bypass} do
-      # bypass = Bypass.open(port: bypass.port)
-
       groups = [
         %{
           "id" => "GROUP_DEVOPS_ID",
@@ -243,7 +241,8 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
         OktaDirectory.mock_group_members_list_endpoint(bypass, group["id"], members)
       end)
 
-      assert execute(%{}) == :ok
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
 
       groups = Actors.Group |> Repo.all()
       assert length(groups) == 2
@@ -287,7 +286,8 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
     test "does not crash on endpoint errors", %{bypass: bypass} do
       Bypass.down(bypass)
 
-      assert execute(%{}) == :ok
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
 
       assert Repo.aggregate(Actors.Group, :count) == 0
     end
@@ -337,7 +337,8 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
       OktaDirectory.mock_groups_list_endpoint(bypass, [])
       OktaDirectory.mock_users_list_endpoint(bypass, users)
 
-      assert execute(%{}) == :ok
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
 
       assert updated_identity =
                Repo.get(Domain.Auth.Identity, identity.id)
@@ -666,7 +667,8 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
         one_member
       )
 
-      assert execute(%{}) == :ok
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
 
       assert updated_group = Repo.get(Domain.Actors.Group, group.id)
       assert updated_group.name == "Group:Engineering"
@@ -758,7 +760,8 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
         end)
       end
 
-      assert execute(%{}) == :ok
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
 
       assert updated_provider = Repo.get(Domain.Auth.Provider, provider.id)
       refute updated_provider.last_synced_at
@@ -774,7 +777,8 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
         end)
       end
 
-      assert execute(%{}) == :ok
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
 
       assert updated_provider = Repo.get(Domain.Auth.Provider, provider.id)
       refute updated_provider.last_synced_at
@@ -783,5 +787,40 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
 
       cancel_bypass_expectations_check(bypass)
     end
+
+    test "sends email on failed directory sync", %{
+      account: account,
+      bypass: bypass
+    } do
+      actor = Fixtures.Actors.create_actor(type: :account_admin_user, account: account)
+      _identity = Fixtures.Auth.create_identity(account: account, actor: actor)
+
+      response = %{
+        "errorCode" => "E0000011",
+        "errorSummary" => "Invalid token provided",
+        "errorLink" => "E0000011",
+        "errorId" => "sampleU-5P2FZVslkYBMP_Rsq",
+        "errorCauses" => []
+      }
+
+      for path <- [
+            "api/v1/users",
+            "api/v1/groups"
+          ] do
+        Bypass.stub(bypass, "GET", path, fn conn ->
+          Plug.Conn.send_resp(conn, 401, Jason.encode!(response))
+        end)
+      end
+
+      {:ok, pid} = Task.Supervisor.start_link()
+      assert execute(%{task_supervisor: pid}) == :ok
+
+      assert_email_sent(fn email ->
+        assert email.subject == "Firezone Identity Provider Sync Error"
+        assert email.text_body =~ "failed to sync 1 times"
+      end)
+
+      cancel_bypass_expectations_check(bypass)
+    end
   end
 end

elixir/apps/domain/test/domain/mailer/auth_email_test.exs

@@ -1,6 +1,6 @@
-defmodule Web.Mailer.AuthEmailTest do
-  use Web.ConnCase, async: true
-  import Web.Mailer.AuthEmail
+defmodule Domain.Mailer.AuthEmailTest do
+  use Domain.DataCase, async: true
+  import Domain.Mailer.AuthEmail
 
   setup do
     Domain.Config.put_env_override(:outbound_email_adapter_configured?, true)

elixir/apps/domain/test/domain/mailer/rate_limiter_test.exs

@@ -1,6 +1,6 @@
-defmodule Web.Mailer.RateLimiterTest do
+defmodule Domain.Mailer.RateLimiterTest do
   use ExUnit.Case, async: true
-  import Web.Mailer.RateLimiter
+  import Domain.Mailer.RateLimiter
 
   describe "init/1" do
     test "creates a ETS table" do

elixir/apps/domain/test/domain/mailer/sync_error_email_test.exs

@@ -0,0 +1,37 @@
+defmodule Domain.Mailer.SyncErrorEmailTest do
+  use Domain.DataCase, async: true
+  import Domain.Mailer.SyncEmail
+
+  setup do
+    account = Fixtures.Accounts.create_account()
+    {provider, _bypass} = Fixtures.Auth.start_and_create_okta_provider(account: account)
+
+    %{
+      account: account,
+      provider: provider
+    }
+  end
+
+  describe "sync_error_email/2" do
+    test "should contain sync error info", %{provider: provider} do
+      admin_email = "admin@foo.local"
+      expected_msg = "403 - Forbidden"
+
+      provider =
+        provider
+        |> Domain.Repo.preload(:account)
+        |> set_provider_failure("Error while syncing")
+        |> set_provider_failure(expected_msg)
+
+      email_body = sync_error_email(provider, admin_email)
+
+      assert email_body.text_body =~ "2 times"
+      assert email_body.text_body =~ expected_msg
+    end
+  end
+
+  defp set_provider_failure(provider, message) do
+    Domain.Auth.Provider.Changeset.sync_failed(provider, message)
+    |> Domain.Repo.update!()
+  end
+end

elixir/apps/domain/test/domain/mailer_test.exs

@@ -1,6 +1,6 @@
-defmodule Web.MailerTest do
+defmodule Domain.MailerTest do
   use ExUnit.Case, async: true
-  import Web.Mailer
+  import Domain.Mailer
 
   describe "deliver_with_rate_limit/2" do
     test "delivers email with rate limit" do

elixir/apps/domain/test/support/data_case.ex

@@ -20,6 +20,7 @@ defmodule Domain.DataCase do
     quote do
       import Ecto
       import Ecto.Changeset
+      import Swoosh.TestAssertions
       import Domain.DataCase
 
       alias Domain.Repo

elixir/apps/domain/test/support/mailer/mailer_test_adapter.ex

@@ -1,4 +1,4 @@
-defmodule Web.Mailer.TestAdapter do
+defmodule Domain.Mailer.TestAdapter do
   use Swoosh.Adapter
 
   @impl true

elixir/apps/web/.formatter.exs

@@ -1,7 +1,10 @@
 [
   import_deps: [:phoenix, :phoenix_live_view],
   plugins: [Phoenix.LiveView.HTMLFormatter],
-  inputs: ["*.{xml.heex,html.heex,ex,exs}", "{config,lib,test}/**/*.{xml.heex,html.heex,ex,exs}"],
+  inputs: [
+    "*.{xml.heex,html.heex,ex,exs}",
+    "{config,lib,test}/**/*.{xml.heex,html.heex,ex,exs}"
+  ],
   locals_without_parens: [
     assert_authenticated: 2,
     assert_unauthenticated: 1,

elixir/apps/web/lib/web/application.ex

@@ -8,8 +8,6 @@ defmodule Web.Application do
     _ = OpentelemetryPhoenix.setup(adapter: :cowboy2)
 
     children = [
-      Web.Mailer,
-      Web.Mailer.RateLimiter,
       Web.Endpoint
     ]
 

elixir/apps/web/lib/web/components/core_components.ex

@@ -1044,16 +1044,11 @@ defmodule Web.CoreComponents do
   end
 
   def get_identity_email(identity) do
-    provider_email(identity) || identity.provider_identifier
+    Domain.Auth.get_identity_email(identity)
   end
 
   def identity_has_email?(identity) do
-    not is_nil(provider_email(identity)) or identity.provider.adapter == :email or
-      identity.provider_identifier =~ "@"
-  end
-
-  defp provider_email(identity) do
-    get_in(identity.provider_state, ["userinfo", "email"])
+    Domain.Auth.identity_has_email?(identity)
   end
 
   attr :account, :any, required: true

elixir/apps/web/lib/web/controllers/auth_controller.ex

@@ -159,14 +159,14 @@ defmodule Web.AuthController do
     # attacks where you can trick user into logging in into an attacker account.
     fragment = identity.provider_virtual_state.fragment
 
-    Web.Mailer.AuthEmail.sign_in_link_email(
+    Domain.Mailer.AuthEmail.sign_in_link_email(
       identity,
       nonce,
       conn.assigns.user_agent,
       conn.remote_ip,
       redirect_params
     )
-    |> Web.Mailer.deliver_with_rate_limit(
+    |> Domain.Mailer.deliver_with_rate_limit(
       rate_limit_key: {:sign_in_link, identity.id},
       rate_limit: 3,
       rate_limit_interval: :timer.minutes(5)
@@ -207,7 +207,7 @@ defmodule Web.AuthController do
       with {:ok, provider} <- Domain.Auth.fetch_active_provider_by_id(provider_id),
            {:ok, identity, encoded_fragment} <-
              Domain.Auth.sign_in(provider, identity_id, nonce, secret, context) do
-        :ok = Web.Mailer.RateLimiter.reset_rate_limit({:sign_in_link, identity.id})
+        :ok = Domain.Mailer.RateLimiter.reset_rate_limit({:sign_in_link, identity.id})
         Web.Auth.signed_in(conn, provider, identity, context, encoded_fragment, redirect_params)
       else
         {:error, :not_found} ->

elixir/apps/web/lib/web/live/actors/show.ex

@@ -636,12 +636,12 @@ defmodule Web.Actors.Show do
   def handle_event("send_welcome_email", %{"id" => id}, socket) do
     {:ok, identity} = Auth.fetch_identity_by_id(id, socket.assigns.subject)
 
-    Web.Mailer.AuthEmail.new_user_email(
+    Domain.Mailer.AuthEmail.new_user_email(
       socket.assigns.account,
       identity,
       socket.assigns.subject
     )
-    |> Web.Mailer.deliver_with_rate_limit(
+    |> Domain.Mailer.deliver_with_rate_limit(
       rate_limit: 3,
       rate_limit_key: {:welcome_email, identity.id},
       rate_limit_interval: :timer.minutes(3)

elixir/apps/web/lib/web/live/actors/users/new_identity.ex

@@ -111,12 +111,12 @@ defmodule Web.Actors.Users.NewIdentity do
              socket.assigns.subject
            ) do
       if socket.assigns.provider.adapter == :email do
-        Web.Mailer.AuthEmail.new_user_email(
+        Domain.Mailer.AuthEmail.new_user_email(
           socket.assigns.account,
           identity,
           socket.assigns.subject
         )
-        |> Web.Mailer.deliver()
+        |> Domain.Mailer.deliver()
       end
 
       socket = push_navigate(socket, to: next_path(socket))

elixir/apps/web/lib/web/live/settings/api_clients/beta.ex

@@ -62,11 +62,11 @@ defmodule Web.Settings.ApiClients.Beta do
   end
 
   def handle_event("request_access", _params, socket) do
-    Web.Mailer.BetaEmail.rest_api_beta_email(
+    Domain.Mailer.BetaEmail.rest_api_beta_email(
       socket.assigns.account,
       socket.assigns.subject
     )
-    |> Web.Mailer.deliver()
+    |> Domain.Mailer.deliver()
 
     socket =
       socket

elixir/apps/web/lib/web/live/sign_up.ex

@@ -469,13 +469,13 @@ defmodule Web.SignUp do
     |> Ecto.Multi.run(
       :send_email,
       fn _repo, %{account: account, identity: identity} ->
-        Web.Mailer.AuthEmail.sign_up_link_email(
+        Domain.Mailer.AuthEmail.sign_up_link_email(
           account,
           identity,
           socket.assigns.user_agent,
           socket.assigns.real_ip
         )
-        |> Web.Mailer.deliver_with_rate_limit(
+        |> Domain.Mailer.deliver_with_rate_limit(
           rate_limit_key: {:sign_up_link, String.downcase(identity.provider_identifier)},
           rate_limit: 3,
           rate_limit_interval: :timer.minutes(30)

elixir/apps/web/mix.exs

@@ -48,10 +48,7 @@ defmodule Web.MixProject do
       {:gettext, "~> 0.20"},
       {:remote_ip, "~> 1.0"},
 
-      # CLDR and unit conversions
-      {:ex_cldr_dates_times, "~> 2.13"},
-      {:ex_cldr_numbers, "~> 2.31"},
-      {:ex_cldr, "~> 2.40"},
+      # Unit conversions
       {:tzdata, "~> 1.1"},
       {:sizeable, "~> 1.0"},
 
@@ -65,11 +62,6 @@ defmodule Web.MixProject do
       {:recon, "~> 2.5"},
       {:observer_cli, "~> 1.7"},
 
-      # Mailer deps
-      {:multipart, "~> 0.4.0"},
-      {:phoenix_swoosh, "~> 1.0"},
-      {:gen_smtp, "~> 1.0"},
-
       # Observability
       {:opentelemetry_telemetry, "~> 1.1.1", override: true},
       {:opentelemetry_cowboy, "~> 0.3"},

elixir/apps/web/test/web/controllers/auth_controller_test.exs

@@ -779,7 +779,7 @@ defmodule Web.AuthControllerTest do
       email_secret: email_secret
     } do
       key = {:sign_in_link, identity.id}
-      Web.Mailer.RateLimiter.rate_limit(key, 3, 60_000, fn -> :ok end)
+      Domain.Mailer.RateLimiter.rate_limit(key, 3, 60_000, fn -> :ok end)
 
       conn =
         conn
@@ -791,7 +791,7 @@ defmodule Web.AuthControllerTest do
       assert conn.assigns.flash == %{}
       assert redirected_to(conn) == ~p"/#{account}/sites"
 
-      refute :ets.tab2list(Web.Mailer.RateLimiter.ETS)
+      refute :ets.tab2list(Domain.Mailer.RateLimiter.ETS)
              |> Enum.any?(fn {ets_key, _, _} -> ets_key == key end)
     end
   end

elixir/config/config.exs

@@ -100,6 +100,8 @@ config :domain, docker_registry: "us-east1-docker.pkg.dev/firezone-staging/firez
 
 config :domain, outbound_email_adapter_configured?: false
 
+config :domain, web_external_url: "http://localhost:13000"
+
 ###############################
 ##### Web #####################
 ###############################
@@ -213,7 +215,7 @@ config :phoenix, :json_library, Jason
 
 config :swoosh, :api_client, Swoosh.ApiClient.Finch
 
-config :web, Web.Mailer,
+config :domain, Domain.Mailer,
   adapter: Domain.Mailer.NoopAdapter,
   from_email: "test@firez.one"
 

elixir/config/dev.exs

@@ -69,7 +69,7 @@ config :phoenix_live_reload, :dirs, [
 config :web, Web.Plugs.SecureHeaders,
   csp_policy: [
     "default-src 'self' 'nonce-${nonce}' https://api-js.mixpanel.com",
-    "img-src 'self' data: https://www.gravatar.com https://track.hubspot.com",
+    "img-src 'self' data: https://www.gravatar.com https://track.hubspot.com https://www.firezone.dev",
     "style-src 'self' 'unsafe-inline'",
     "script-src 'self' 'unsafe-inline' http://cdn.mxpnl.com http://*.hs-analytics.net https://cdn.tailwindcss.com/"
   ]
@@ -109,7 +109,7 @@ config :phoenix, :stacktrace_depth, 20
 # Initialize plugs at runtime for faster development compilation
 config :phoenix, :plug_init_mode, :runtime
 
-config :web, Web.Mailer, adapter: Swoosh.Adapters.Local
+config :domain, Domain.Mailer, adapter: Swoosh.Adapters.Local
 
 config :workos, WorkOS.Client,
   api_key: System.get_env("WORKOS_API_KEY"),

elixir/config/runtime.exs

@@ -73,6 +73,8 @@ if config_env() == :prod do
 
   config :domain, outbound_email_adapter_configured?: !!compile_config!(:outbound_email_adapter)
 
+  config :domain, web_external_url: compile_config!(:web_external_url)
+
   # Enable background jobs only on dedicated nodes
   config :domain, Domain.Tokens.Jobs.DeleteExpiredTokens,
     enabled: compile_config!(:background_jobs_enabled)
@@ -221,8 +223,8 @@ if config_env() == :prod do
   config :openid_connect,
     finch_transport_opts: compile_config!(:http_client_ssl_opts)
 
-  config :web,
-         Web.Mailer,
+  config :domain,
+         Domain.Mailer,
          [
            adapter: compile_config!(:outbound_email_adapter),
            from_email: compile_config!(:outbound_email_from)

elixir/config/test.exs

@@ -28,6 +28,8 @@ config :domain, Domain.GoogleCloudPlatform, service_account_email: "foo@iam.exam
 
 config :domain, Domain.Telemetry.GoogleCloudMetricsReporter, project_id: "fz-test"
 
+config :domain, web_external_url: "http://localhost:13100"
+
 ###############################
 ##### Web #####################
 ###############################
@@ -57,7 +59,7 @@ config :api, API.Endpoint,
 ###############################
 ##### Third-party configs #####
 ###############################
-config :web, Web.Mailer, adapter: Web.Mailer.TestAdapter
+config :domain, Domain.Mailer, adapter: Domain.Mailer.TestAdapter
 
 config :logger, level: :warning