fix(portal): Gracefully handle dir sync error responses (#8608)

When calling the various directory sync endpoints, we had error cases
that matched a few of the possible error scenarios in an appropriate way
by returning either `{:error, :retry_later}` or the `{:error, ...}`
tuples.

However, as we've recently learned in [this
thread](https://firezonehq.slack.com/archives/C069H865MHP/p1743521884037159),
it's possible for identity provider APIs to return all kinds of bogus
data here, and we need a more defensive approach.

The specific issue this PR addresses is the case where we receive a
`2xx` response, but without the expected JSON key in the response body.
That will result in the `list*` functions returning an empty list, which
the calling code paths then use to soft-delete all existing record types
in the DB.

This is wrong. If the JSON response is missing a key we're expecting, we
instead log a warning and return `{:error, :retry_later}`. It's
currently unknown when exactly this happens and why, but with better
monitoring here we'll have a much better picture as to why.

elixir/apps/domain/lib/domain/auth/adapters/google_workspace/api_client.ex

@@ -4,6 +4,7 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.APIClient do
   or they will not return you pagination cursor 🫠.
   """
   use Supervisor
+  require Logger
 
   @pool_name __MODULE__.Finch
 
@@ -173,16 +174,31 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.APIClient do
   defp list(uri, api_token, key) do
     request = Finch.build(:get, uri, [{"Authorization", "Bearer #{api_token}"}])
 
-    with {:ok, %Finch.Response{body: response, status: status}} when status in 200..299 <-
+    with {:ok, %Finch.Response{body: response, status: 200}} <-
            Finch.request(request, @pool_name),
          {:ok, json_response} <- Jason.decode(response),
-         {:ok, list} <- Map.fetch(json_response, key) do
+         {:ok, list} when is_list(list) <- Map.fetch(json_response, key) do
       {:ok, list, json_response["nextPageToken"]}
     else
-      {:ok, %Finch.Response{status: status}} when status in 500..599 ->
+      {:ok, %Finch.Response{status: status} = response} when status in 201..299 ->
+        Logger.warning("API request succeeded with unexpected 2xx status #{status}",
+          response: inspect(response)
+        )
+
         {:error, :retry_later}
 
-      {:ok, %Finch.Response{body: response, status: status}} ->
+      {:ok, %Finch.Response{status: status} = response} when status in 300..399 ->
+        Logger.warning("API request succeeded with unexpected 3xx status #{status}",
+          response: inspect(response)
+        )
+
+        {:error, :retry_later}
+
+      {:ok, %Finch.Response{body: response, status: status}} when status in 400..499 ->
+        Logger.error("API request failed with 4xx status #{status}",
+          response: inspect(response)
+        )
+
         case Jason.decode(response) do
           {:ok, json_response} ->
             {:error, {status, json_response}}
@@ -191,10 +207,32 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.APIClient do
             {:error, {status, response}}
         end
 
+      {:ok, %Finch.Response{status: status} = response} when status in 500..599 ->
+        Logger.error("API request failed with 5xx status #{status}",
+          response: inspect(response)
+        )
+
+        {:error, :retry_later}
+
+      {:ok, not_a_list} when not is_list(not_a_list) ->
+        Logger.error("API request failed with unexpected data format",
+          uri: inspect(uri),
+          key: key
+        )
+
+        {:error, :retry_later}
+
       :error ->
-        {:ok, [], nil}
+        Logger.error("API response did not contain expected key",
+          uri: inspect(uri),
+          key: key
+        )
+
+        {:error, :retry_later}
 
       other ->
+        Logger.error("Unexpected response from API", response: inspect(other))
+
         other
     end
   end

elixir/apps/domain/lib/domain/auth/adapters/microsoft_entra/api_client.ex

@@ -1,5 +1,6 @@
 defmodule Domain.Auth.Adapters.MicrosoftEntra.APIClient do
   use Supervisor
+  require Logger
 
   @pool_name __MODULE__.Finch
 
@@ -132,16 +133,31 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.APIClient do
   defp list(uri, api_token) do
     request = Finch.build(:get, uri, [{"Authorization", "Bearer #{api_token}"}])
 
-    with {:ok, %Finch.Response{body: response, status: status}} when status in 200..299 <-
+    with {:ok, %Finch.Response{body: response, status: 200}} <-
            Finch.request(request, @pool_name),
          {:ok, json_response} <- Jason.decode(response),
-         {:ok, list} <- Map.fetch(json_response, "value") do
+         {:ok, list} when is_list(list) <- Map.fetch(json_response, "value") do
       {:ok, list, json_response["@odata.nextLink"]}
     else
-      {:ok, %Finch.Response{status: status}} when status in 500..599 ->
+      {:ok, %Finch.Response{status: status} = response} when status in 201..299 ->
+        Logger.warning("API request succeeded with unexpected 2xx status #{status}",
+          response: inspect(response)
+        )
+
         {:error, :retry_later}
 
-      {:ok, %Finch.Response{body: response, status: status}} ->
+      {:ok, %Finch.Response{status: status} = response} when status in 300..399 ->
+        Logger.warning("API request succeeded with unexpected 3xx status #{status}",
+          response: inspect(response)
+        )
+
+        {:error, :retry_later}
+
+      {:ok, %Finch.Response{body: response, status: status}} when status in 400..499 ->
+        Logger.error("API request failed with 4xx status #{status}",
+          response: inspect(response)
+        )
+
         case Jason.decode(response) do
           {:ok, json_response} ->
             {:error, {status, json_response}}
@@ -150,10 +166,30 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.APIClient do
             {:error, {status, response}}
         end
 
+      {:ok, %Finch.Response{status: status} = response} when status in 500..599 ->
+        Logger.error("API request failed with 5xx status #{status}",
+          response: inspect(response)
+        )
+
+        {:error, :retry_later}
+
+      {:ok, not_a_list} when not is_list(not_a_list) ->
+        Logger.error("API request failed with unexpected data format",
+          uri: inspect(uri)
+        )
+
+        {:error, :retry_later}
+
       :error ->
-        {:ok, [], nil}
+        Logger.error("API response did not contain expected 'value' key",
+          uri: inspect(uri)
+        )
+
+        {:error, :retry_later}
 
       other ->
+        Logger.error("Unexpected response from API", response: inspect(other))
+
         other
     end
   end

elixir/apps/domain/lib/domain/auth/adapters/okta/api_client.ex

@@ -1,5 +1,6 @@
 defmodule Domain.Auth.Adapters.Okta.APIClient do
   use Supervisor
+  require Logger
 
   @pool_name __MODULE__.Finch
 
@@ -114,15 +115,30 @@ defmodule Domain.Auth.Adapters.Okta.APIClient do
     # Crude request throttle, revisit for https://github.com/firezone/firezone/issues/6793
     throttle()
 
-    with {:ok, %Finch.Response{headers: headers, body: response, status: status}}
-         when status in 200..299 <- Finch.request(request, @pool_name),
-         {:ok, list} <- Jason.decode(response) do
+    with {:ok, %Finch.Response{headers: headers, body: response, status: 200}} <-
+           Finch.request(request, @pool_name),
+         {:ok, list} when is_list(list) <- Jason.decode(response) do
       {:ok, list, fetch_next_link(headers)}
     else
-      {:ok, %Finch.Response{status: status}} when status in 500..599 ->
+      {:ok, %Finch.Response{status: status} = response} when status in 201..299 ->
+        Logger.warning("API request succeeded with unexpected 2xx status #{status}",
+          response: inspect(response)
+        )
+
         {:error, :retry_later}
 
-      {:ok, %Finch.Response{body: response, status: status}} ->
+      {:ok, %Finch.Response{status: status} = response} when status in 300..399 ->
+        Logger.warning("API request succeeded with unexpected 3xx status #{status}",
+          response: inspect(response)
+        )
+
+        {:error, :retry_later}
+
+      {:ok, %Finch.Response{body: response, status: status}} when status in 400..499 ->
+        Logger.error("API request failed with 4xx status #{status}",
+          response: inspect(response)
+        )
+
         case Jason.decode(response) do
           {:ok, json_response} ->
             {:error, {status, json_response}}
@@ -131,7 +147,23 @@ defmodule Domain.Auth.Adapters.Okta.APIClient do
             {:error, {status, response}}
         end
 
+      {:ok, %Finch.Response{status: status} = response} when status in 500..599 ->
+        Logger.error("API request failed with 5xx status #{status}",
+          response: inspect(response)
+        )
+
+        {:error, :retry_later}
+
+      {:ok, not_a_list} when not is_list(not_a_list) ->
+        Logger.error("API request failed with unexpected data format",
+          uri: inspect(uri)
+        )
+
+        {:error, :retry_later}
+
       other ->
+        Logger.error("Unexpected response from API", response: inspect(other))
+
         other
     end
   end

elixir/apps/domain/test/domain/auth/adapters/google_workspace/api_client_test.exs

@@ -7,7 +7,7 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.APIClientTest do
     test "returns list of users" do
       api_token = Ecto.UUID.generate()
       bypass = Bypass.open()
-      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass)
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, 200)
       assert {:ok, users} = list_users(api_token)
 
       assert length(users) == 4
@@ -47,13 +47,81 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.APIClientTest do
       Bypass.down(bypass)
       assert list_users(api_token) == {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, 201)
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, 301)
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(
+        bypass,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_users(api_token) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, 500)
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds without expected JSON keys" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{})
+      )
+
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"users" => "invalid data"})
+      )
+
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, 200, "invalid json")
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} = list_users(api_token)
+    end
   end
 
   describe "list_organization_units/1" do
     test "returns list of organization units" do
       api_token = Ecto.UUID.generate()
       bypass = Bypass.open()
-      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass)
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, 200)
       assert {:ok, organization_units} = list_organization_units(api_token)
 
       assert length(organization_units) == 1
@@ -78,13 +146,83 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.APIClientTest do
       assert list_organization_units(api_token) ==
                {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, 201)
+      assert list_organization_units(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, 301)
+      assert list_organization_units(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(
+        bypass,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_organization_units(api_token) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, 500)
+      assert list_organization_units(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds without expected JSON keys" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{})
+      )
+
+      assert list_organization_units(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"organization_units" => "invalid data"})
+      )
+
+      assert list_organization_units(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, 200, "invalid json")
+
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} =
+               list_organization_units(api_token)
+    end
   end
 
   describe "list_groups/1" do
     test "returns list of groups" do
       api_token = Ecto.UUID.generate()
       bypass = Bypass.open()
-      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass)
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, 200)
       assert {:ok, groups} = list_groups(api_token)
 
       assert length(groups) == 3
@@ -111,6 +249,74 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.APIClientTest do
       Bypass.down(bypass)
       assert list_groups(api_token) == {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, 201)
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, 301)
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(
+        bypass,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_groups(api_token) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, 500)
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds without expected JSON keys" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{})
+      )
+
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"groups" => "invalid data"})
+      )
+
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, 200, "invalid json")
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} = list_groups(api_token)
+    end
   end
 
   describe "list_group_members/1" do
@@ -119,7 +325,7 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.APIClientTest do
       group_id = Ecto.UUID.generate()
 
       bypass = Bypass.open()
-      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(bypass, group_id)
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(bypass, group_id, 200)
       assert {:ok, members} = list_group_members(api_token, group_id)
 
       assert length(members) == 2
@@ -145,5 +351,91 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.APIClientTest do
       assert list_group_members(api_token, group_id) ==
                {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(bypass, group_id, 201)
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(bypass, group_id, 301)
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_group_members(api_token, group_id) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(bypass, group_id, 500)
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds without expected JSON keys" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        200,
+        Jason.encode!(%{})
+      )
+
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        200,
+        Jason.encode!(%{"group_members" => "invalid data"})
+      )
+
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        200,
+        "invalid json"
+      )
+
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} =
+               list_group_members(api_token, group_id)
+    end
   end
 end

elixir/apps/domain/test/domain/auth/adapters/google_workspace/jobs/sync_directory_test.exs

@@ -36,9 +36,25 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.Jobs.SyncDirectoryTest do
       bypass = Bypass.open()
 
       GoogleWorkspaceDirectory.override_endpoint_url("http://localhost:#{bypass.port}/")
-      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, [])
-      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, [])
-      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, [])
+
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"groups" => []})
+      )
+
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"organizationUnits" => []})
+      )
+
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"users" => []})
+      )
+
       GoogleWorkspaceDirectory.mock_token_endpoint(bypass)
 
       {:ok, pid} = Task.Supervisor.start_link()
@@ -80,9 +96,24 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.Jobs.SyncDirectoryTest do
       bypass = Bypass.open()
 
       GoogleWorkspaceDirectory.override_endpoint_url("http://localhost:#{bypass.port}/")
-      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, [])
-      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, [])
-      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, [])
+
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"groups" => []})
+      )
+
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"organizationUnits" => []})
+      )
+
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"users" => []})
+      )
 
       provider
       |> Ecto.Changeset.change(
@@ -246,13 +277,34 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.Jobs.SyncDirectoryTest do
       ]
 
       GoogleWorkspaceDirectory.override_endpoint_url("http://localhost:#{bypass.port}/")
-      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, groups)
-      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, organization_units)
-      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, users)
+
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"groups" => groups})
+      )
+
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"organizationUnits" => organization_units})
+      )
+
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"users" => users})
+      )
+
       GoogleWorkspaceDirectory.mock_token_endpoint(bypass)
 
       Enum.each(groups, fn group ->
-        GoogleWorkspaceDirectory.mock_group_members_list_endpoint(bypass, group["id"], members)
+        GoogleWorkspaceDirectory.mock_group_members_list_endpoint(
+          bypass,
+          group["id"],
+          200,
+          Jason.encode!(%{"members" => members})
+        )
       end)
 
       {:ok, pid} = Task.Supervisor.start_link()
@@ -342,9 +394,25 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.Jobs.SyncDirectoryTest do
         )
 
       GoogleWorkspaceDirectory.override_endpoint_url("http://localhost:#{bypass.port}/")
-      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, [])
-      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, [])
-      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, users)
+
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"groups" => []})
+      )
+
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"organizationUnits" => []})
+      )
+
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"users" => users})
+      )
+
       GoogleWorkspaceDirectory.mock_token_endpoint(bypass)
 
       {:ok, pid} = Task.Supervisor.start_link()
@@ -572,13 +640,40 @@ defmodule Domain.Auth.Adapters.GoogleWorkspace.Jobs.SyncDirectoryTest do
       :ok = Phoenix.PubSub.subscribe(Domain.PubSub, "sessions:#{deleted_identity_token.id}")
 
       GoogleWorkspaceDirectory.override_endpoint_url("http://localhost:#{bypass.port}/")
-      GoogleWorkspaceDirectory.mock_groups_list_endpoint(bypass, groups)
-      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(bypass, organization_units)
-      GoogleWorkspaceDirectory.mock_users_list_endpoint(bypass, users)
+
+      GoogleWorkspaceDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"groups" => groups})
+      )
+
+      GoogleWorkspaceDirectory.mock_organization_units_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"organizationUnits" => organization_units})
+      )
+
+      GoogleWorkspaceDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"users" => users})
+      )
+
       GoogleWorkspaceDirectory.mock_token_endpoint(bypass)
 
-      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(bypass, "GROUP_ID1", two_members)
-      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(bypass, "GROUP_ID2", one_member)
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(
+        bypass,
+        "GROUP_ID1",
+        200,
+        Jason.encode!(%{"members" => two_members})
+      )
+
+      GoogleWorkspaceDirectory.mock_group_members_list_endpoint(
+        bypass,
+        "GROUP_ID2",
+        200,
+        Jason.encode!(%{"members" => one_member})
+      )
 
       {:ok, pid} = Task.Supervisor.start_link()
       assert execute(%{task_supervisor: pid}) == :ok

elixir/apps/domain/test/domain/auth/adapters/microsoft_entra/api_client_test.exs

@@ -7,7 +7,7 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.APIClientTest do
     test "returns list of users" do
       api_token = Ecto.UUID.generate()
       bypass = Bypass.open()
-      MicrosoftEntraDirectory.mock_users_list_endpoint(bypass)
+      MicrosoftEntraDirectory.mock_users_list_endpoint(bypass, 200)
       assert {:ok, users} = list_users(api_token)
 
       assert length(users) == 3
@@ -54,13 +54,81 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.APIClientTest do
       Bypass.down(bypass)
       assert list_users(api_token) == {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_users_list_endpoint(bypass, 201)
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_users_list_endpoint(bypass, 301)
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_users_list_endpoint(
+        bypass,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_users(api_token) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_users_list_endpoint(bypass, 500)
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds without expected JSON keys" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{})
+      )
+
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"users" => "invalid data"})
+      )
+
+      assert list_users(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_users_list_endpoint(bypass, 200, "invalid json")
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} = list_users(api_token)
+    end
   end
 
   describe "list_groups/1" do
     test "returns list of groups" do
       api_token = Ecto.UUID.generate()
       bypass = Bypass.open()
-      MicrosoftEntraDirectory.mock_groups_list_endpoint(bypass)
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(bypass, 200)
       assert {:ok, groups} = list_groups(api_token)
 
       assert length(groups) == 3
@@ -87,6 +155,76 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.APIClientTest do
       Bypass.down(bypass)
       assert list_groups(api_token) == {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(bypass, 201)
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(bypass, 301)
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(
+        bypass,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_groups(api_token) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(bypass, 500)
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds without expected JSON keys" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{})
+      )
+
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"groups" => "invalid data"})
+      )
+
+      assert list_groups(api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(bypass, 200, "invalid json")
+
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} =
+               list_groups(api_token)
+    end
   end
 
   describe "list_group_members/1" do
@@ -95,7 +233,7 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.APIClientTest do
       group_id = Ecto.UUID.generate()
 
       bypass = Bypass.open()
-      MicrosoftEntraDirectory.mock_group_members_list_endpoint(bypass, group_id)
+      MicrosoftEntraDirectory.mock_group_members_list_endpoint(bypass, group_id, 200)
       assert {:ok, members} = list_group_members(api_token, group_id)
 
       assert length(members) == 3
@@ -121,5 +259,91 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.APIClientTest do
       assert list_group_members(api_token, group_id) ==
                {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_group_members_list_endpoint(bypass, group_id, 201)
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_group_members_list_endpoint(bypass, group_id, 301)
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_group_members(api_token, group_id) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      MicrosoftEntraDirectory.mock_group_members_list_endpoint(bypass, group_id, 500)
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds without expected JSON keys" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        200,
+        Jason.encode!(%{})
+      )
+
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        200,
+        Jason.encode!(%{"group_members" => "invalid data"})
+      )
+
+      assert list_group_members(api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+
+      MicrosoftEntraDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        200,
+        "invalid json"
+      )
+
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} =
+               list_group_members(api_token, group_id)
+    end
   end
 end

elixir/apps/domain/test/domain/auth/adapters/microsoft_entra/jobs/sync_directory_test.exs

@@ -77,11 +77,26 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.Jobs.SyncDirectoryTest do
       ]
 
       MicrosoftEntraDirectory.override_endpoint_url("http://localhost:#{bypass.port}/")
-      MicrosoftEntraDirectory.mock_groups_list_endpoint(bypass, groups)
-      MicrosoftEntraDirectory.mock_users_list_endpoint(bypass, users)
+
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"value" => groups})
+      )
+
+      MicrosoftEntraDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"value" => users})
+      )
 
       Enum.each(groups, fn group ->
-        MicrosoftEntraDirectory.mock_group_members_list_endpoint(bypass, group["id"], members)
+        MicrosoftEntraDirectory.mock_group_members_list_endpoint(
+          bypass,
+          group["id"],
+          200,
+          Jason.encode!(%{"value" => members})
+        )
       end)
 
       {:ok, pid} = Task.Supervisor.start_link()
@@ -163,8 +178,18 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.Jobs.SyncDirectoryTest do
         )
 
       MicrosoftEntraDirectory.override_endpoint_url("http://localhost:#{bypass.port}/")
-      MicrosoftEntraDirectory.mock_groups_list_endpoint(bypass, [])
-      MicrosoftEntraDirectory.mock_users_list_endpoint(bypass, users)
+
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"value" => []})
+      )
+
+      MicrosoftEntraDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"value" => users})
+      )
 
       {:ok, pid} = Task.Supervisor.start_link()
       assert execute(%{task_supervisor: pid}) == :ok
@@ -319,19 +344,30 @@ defmodule Domain.Auth.Adapters.MicrosoftEntra.Jobs.SyncDirectoryTest do
       :ok = Domain.Flows.subscribe_to_flow_expiration_events(deleted_identity_flow)
       :ok = Phoenix.PubSub.subscribe(Domain.PubSub, "sessions:#{deleted_identity_token.id}")
 
-      MicrosoftEntraDirectory.mock_groups_list_endpoint(bypass, groups)
-      MicrosoftEntraDirectory.mock_users_list_endpoint(bypass, users)
+      MicrosoftEntraDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"value" => groups})
+      )
+
+      MicrosoftEntraDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"value" => users})
+      )
 
       MicrosoftEntraDirectory.mock_group_members_list_endpoint(
         bypass,
         "GROUP_ALL_ID",
-        two_members
+        200,
+        Jason.encode!(%{"value" => two_members})
       )
 
       MicrosoftEntraDirectory.mock_group_members_list_endpoint(
         bypass,
         "GROUP_ENGINEERING_ID",
-        one_member
+        200,
+        Jason.encode!(%{"value" => one_member})
       )
 
       {:ok, pid} = Task.Supervisor.start_link()

elixir/apps/domain/test/domain/auth/adapters/okta/api_client_test.exs

@@ -8,7 +8,7 @@ defmodule Domain.Auth.Adapters.Okta.APIClientTest do
       api_token = Ecto.UUID.generate()
       bypass = Bypass.open()
       api_base_url = "http://localhost:#{bypass.port}/"
-      OktaDirectory.mock_users_list_endpoint(bypass)
+      OktaDirectory.mock_users_list_endpoint(bypass, 200)
 
       assert {:ok, users} = list_users(api_base_url, api_token)
       assert length(users) == 2
@@ -41,6 +41,69 @@ defmodule Domain.Auth.Adapters.Okta.APIClientTest do
       assert list_users(api_base_url, api_token) ==
                {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_users_list_endpoint(bypass, 201)
+      assert list_users(api_base_url, api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_users_list_endpoint(bypass, 301)
+      assert list_users(api_base_url, api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+
+      OktaDirectory.mock_users_list_endpoint(
+        bypass,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_users(api_base_url, api_token) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_users_list_endpoint(bypass, 500)
+      assert list_users(api_base_url, api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+
+      OktaDirectory.mock_users_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"invalid" => "format"})
+      )
+
+      assert list_users(api_base_url, api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_users_list_endpoint(bypass, 200, "invalid json")
+
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} =
+               list_users(api_base_url, api_token)
+    end
   end
 
   describe "list_groups/1" do
@@ -48,7 +111,7 @@ defmodule Domain.Auth.Adapters.Okta.APIClientTest do
       api_token = Ecto.UUID.generate()
       bypass = Bypass.open()
       api_base_url = "http://localhost:#{bypass.port}/"
-      OktaDirectory.mock_groups_list_endpoint(bypass)
+      OktaDirectory.mock_groups_list_endpoint(bypass, 200)
 
       assert {:ok, groups} = list_groups(api_base_url, api_token)
       assert length(groups) == 4
@@ -80,6 +143,69 @@ defmodule Domain.Auth.Adapters.Okta.APIClientTest do
       assert list_groups(api_base_url, api_token) ==
                {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_groups_list_endpoint(bypass, 201)
+      assert list_groups(api_base_url, api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_groups_list_endpoint(bypass, 301)
+      assert list_groups(api_base_url, api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+
+      OktaDirectory.mock_groups_list_endpoint(
+        bypass,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_groups(api_base_url, api_token) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_groups_list_endpoint(bypass, 500)
+      assert list_groups(api_base_url, api_token) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+
+      OktaDirectory.mock_groups_list_endpoint(
+        bypass,
+        200,
+        Jason.encode!(%{"invalid" => "format"})
+      )
+
+      assert list_groups(api_base_url, api_token) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_groups_list_endpoint(bypass, 200, "invalid json")
+
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} =
+               list_groups(api_base_url, api_token)
+    end
   end
 
   describe "list_group_members/1" do
@@ -89,7 +215,7 @@ defmodule Domain.Auth.Adapters.Okta.APIClientTest do
 
       bypass = Bypass.open()
       api_base_url = "http://localhost:#{bypass.port}/"
-      OktaDirectory.mock_group_members_list_endpoint(bypass, group_id)
+      OktaDirectory.mock_group_members_list_endpoint(bypass, group_id, 200)
 
       assert {:ok, members} = list_group_members(api_base_url, api_token, group_id)
 
@@ -117,5 +243,82 @@ defmodule Domain.Auth.Adapters.Okta.APIClientTest do
       assert list_group_members(api_base_url, api_token, group_id) ==
                {:error, %Mint.TransportError{reason: :econnrefused}}
     end
+
+    test "returns retry_later when api responds with unexpected 2xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_group_members_list_endpoint(bypass, group_id, 201)
+      assert list_group_members(api_base_url, api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected 3xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_group_members_list_endpoint(bypass, group_id, 301)
+      assert list_group_members(api_base_url, api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with 4xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+
+      OktaDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        400,
+        Jason.encode!(%{"error" => %{"code" => 400, "message" => "Bad Request"}})
+      )
+
+      assert list_group_members(api_base_url, api_token, group_id) ==
+               {:error, {400, %{"error" => %{"code" => 400, "message" => "Bad Request"}}}}
+    end
+
+    test "returns retry_later when api responds with 5xx status" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+      OktaDirectory.mock_group_members_list_endpoint(bypass, group_id, 500)
+      assert list_group_members(api_base_url, api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns retry_later when api responds with unexpected data format" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+
+      OktaDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        200,
+        Jason.encode!(%{"invalid" => "data"})
+      )
+
+      assert list_group_members(api_base_url, api_token, group_id) == {:error, :retry_later}
+    end
+
+    test "returns error when api responds with invalid JSON" do
+      api_token = Ecto.UUID.generate()
+      group_id = Ecto.UUID.generate()
+      bypass = Bypass.open()
+      api_base_url = "http://localhost:#{bypass.port}/"
+
+      OktaDirectory.mock_group_members_list_endpoint(
+        bypass,
+        group_id,
+        200,
+        "invalid json"
+      )
+
+      assert {:error, %Jason.DecodeError{data: "invalid json"}} =
+               list_group_members(api_base_url, api_token, group_id)
+    end
   end
 end

elixir/apps/domain/test/domain/auth/adapters/okta/jobs/sync_directory_test.exs

@@ -234,11 +234,16 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
         }
       ]
 
-      OktaDirectory.mock_groups_list_endpoint(bypass, groups)
-      OktaDirectory.mock_users_list_endpoint(bypass, users)
+      OktaDirectory.mock_groups_list_endpoint(bypass, 200, Jason.encode!(groups))
+      OktaDirectory.mock_users_list_endpoint(bypass, 200, Jason.encode!(users))
 
       Enum.each(groups, fn group ->
-        OktaDirectory.mock_group_members_list_endpoint(bypass, group["id"], members)
+        OktaDirectory.mock_group_members_list_endpoint(
+          bypass,
+          group["id"],
+          200,
+          Jason.encode!(members)
+        )
       end)
 
       {:ok, pid} = Task.Supervisor.start_link()
@@ -334,8 +339,8 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
           provider_identifier: "USER_JDOE_ID"
         )
 
-      OktaDirectory.mock_groups_list_endpoint(bypass, [])
-      OktaDirectory.mock_users_list_endpoint(bypass, users)
+      OktaDirectory.mock_groups_list_endpoint(bypass, 200, Jason.encode!([]))
+      OktaDirectory.mock_users_list_endpoint(bypass, 200, Jason.encode!(users))
 
       {:ok, pid} = Task.Supervisor.start_link()
       assert execute(%{task_supervisor: pid}) == :ok
@@ -652,19 +657,21 @@ defmodule Domain.Auth.Adapters.Okta.Jobs.SyncDirectoryTest do
       :ok = Domain.Flows.subscribe_to_flow_expiration_events(deleted_identity_flow)
       :ok = Phoenix.PubSub.subscribe(Domain.PubSub, "sessions:#{deleted_identity_token.id}")
 
-      OktaDirectory.mock_groups_list_endpoint(bypass, groups)
-      OktaDirectory.mock_users_list_endpoint(bypass, users)
+      OktaDirectory.mock_groups_list_endpoint(bypass, 200, Jason.encode!(groups))
+      OktaDirectory.mock_users_list_endpoint(bypass, 200, Jason.encode!(users))
 
       OktaDirectory.mock_group_members_list_endpoint(
         bypass,
         "GROUP_ENGINEERING_ID",
-        two_members
+        200,
+        Jason.encode!(two_members)
       )
 
       OktaDirectory.mock_group_members_list_endpoint(
         bypass,
         "GROUP_DEVOPS_ID",
-        one_member
+        200,
+        Jason.encode!(one_member)
       )
 
       {:ok, pid} = Task.Supervisor.start_link()

elixir/apps/domain/test/support/mocks/google_workspace_directory.ex

@@ -35,14 +35,14 @@ defmodule Domain.Mocks.GoogleWorkspaceDirectory do
     bypass
   end
 
-  def mock_users_list_endpoint(bypass, users \\ nil) do
+  def mock_users_list_endpoint(bypass, status, resp \\ nil) do
     users_list_endpoint_path = "/admin/directory/v1/users"
 
-    resp = %{
-      "kind" => "admin#directory#users",
-      "users" =>
-        users ||
-          [
+    resp =
+      resp ||
+        Jason.encode!(%{
+          "kind" => "admin#directory#users",
+          "users" => [
             %{
               "agreedToTerms" => true,
               "archived" => false,
@@ -211,14 +211,14 @@ defmodule Domain.Mocks.GoogleWorkspaceDirectory do
                 "https://lh3.google.com/ao/AP2z2aWvm9JM99oCFZ1TVOJgQZlmZdMMYNr7w9G0jZApdTuLHfAueGFb_XzgTvCNRhGw=s96-c"
             }
           ]
-    }
+        })
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", users_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     override_endpoint_url("http://localhost:#{bypass.port}/")
@@ -226,15 +226,15 @@ defmodule Domain.Mocks.GoogleWorkspaceDirectory do
     bypass
   end
 
-  def mock_organization_units_list_endpoint(bypass, org_units \\ nil) do
+  def mock_organization_units_list_endpoint(bypass, status, resp \\ nil) do
     org_units_list_endpoint_path = "/admin/directory/v1/customer/my_customer/orgunits"
 
-    resp = %{
-      "kind" => "admin#directory#org_units",
-      "etag" => "\"FwDC5ZsOozt9qI9yuJfiMqwYO1K-EEG4flsXSov57CY/Y3F7O3B5N0h0C_3Pd3OMifRNUVc\"",
-      "organizationUnits" =>
-        org_units ||
-          [
+    resp =
+      resp ||
+        Jason.encode!(%{
+          "kind" => "admin#directory#org_units",
+          "etag" => "\"FwDC5ZsOozt9qI9yuJfiMqwYO1K-EEG4flsXSov57CY/Y3F7O3B5N0h0C_3Pd3OMifRNUVc\"",
+          "organizationUnits" => [
             %{
               "kind" => "admin#directory#orgUnit",
               "name" => "Engineering",
@@ -247,14 +247,14 @@ defmodule Domain.Mocks.GoogleWorkspaceDirectory do
               "parentOrgUnitPath" => "/"
             }
           ]
-    }
+        })
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", org_units_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     override_endpoint_url("http://localhost:#{bypass.port}/")
@@ -262,15 +262,15 @@ defmodule Domain.Mocks.GoogleWorkspaceDirectory do
     bypass
   end
 
-  def mock_groups_list_endpoint(bypass, groups \\ nil) do
+  def mock_groups_list_endpoint(bypass, status, resp \\ nil) do
     groups_list_endpoint_path = "/admin/directory/v1/groups"
 
-    resp = %{
-      "kind" => "admin#directory#groups",
-      "etag" => "\"FwDC5ZsOozt9qI9yuJfiMqwYO1K-EEG4flsXSov57CY/Y3F7O3B5N0h0C_3Pd3OMifRNUVc\"",
-      "groups" =>
-        groups ||
-          [
+    resp =
+      resp ||
+        Jason.encode!(%{
+          "kind" => "admin#directory#groups",
+          "etag" => "\"FwDC5ZsOozt9qI9yuJfiMqwYO1K-EEG4flsXSov57CY/Y3F7O3B5N0h0C_3Pd3OMifRNUVc\"",
+          "groups" => [
             %{
               "kind" => "admin#directory#group",
               "id" => "GROUP_ID1",
@@ -314,14 +314,14 @@ defmodule Domain.Mocks.GoogleWorkspaceDirectory do
               ]
             }
           ]
-    }
+        })
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", groups_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     override_endpoint_url("http://localhost:#{bypass.port}/")
@@ -329,15 +329,15 @@ defmodule Domain.Mocks.GoogleWorkspaceDirectory do
     bypass
   end
 
-  def mock_group_members_list_endpoint(bypass, group_id, members \\ nil) do
+  def mock_group_members_list_endpoint(bypass, group_id, status, resp \\ nil) do
     group_members_list_endpoint_path = "/admin/directory/v1/groups/#{group_id}/members"
 
-    resp = %{
-      "kind" => "admin#directory#members",
-      "etag" => "\"XXX\"",
-      "members" =>
-        members ||
-          [
+    resp =
+      resp ||
+        Jason.encode!(%{
+          "kind" => "admin#directory#members",
+          "etag" => "\"XXX\"",
+          "members" => [
             %{
               "kind" => "admin#directory#member",
               "etag" => "\"ET\"",
@@ -384,14 +384,14 @@ defmodule Domain.Mocks.GoogleWorkspaceDirectory do
               "status" => "ACTIVE"
             }
           ]
-    }
+        })
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", group_members_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     override_endpoint_url("http://localhost:#{bypass.port}/")

elixir/apps/domain/test/support/mocks/microsoft_entra_directory.ex

@@ -7,15 +7,15 @@ defmodule Domain.Mocks.MicrosoftEntraDirectory do
     Domain.Config.put_env_override(:domain, MicrosoftEntra.APIClient, config)
   end
 
-  def mock_users_list_endpoint(bypass, users \\ nil) do
+  def mock_users_list_endpoint(bypass, status, resp \\ nil) do
     users_list_endpoint_path = "v1.0/users"
 
-    resp = %{
-      "@odata.context" =>
-        "https://graph.microsoft.com/v1.0/$metadata#users(id,displayName,userPrincipalName,mail,accountEnabled)",
-      "value" =>
-        users ||
-          [
+    resp =
+      resp ||
+        Jason.encode!(%{
+          "@odata.context" =>
+            "https://graph.microsoft.com/v1.0/$metadata#users(id,displayName,userPrincipalName,mail,accountEnabled)",
+          "value" => [
             %{
               "id" => "8FBDDD1B-0E73-4CD0-AD38-2ACEA67814EE",
               "displayName" => "John Doe",
@@ -44,14 +44,14 @@ defmodule Domain.Mocks.MicrosoftEntraDirectory do
               "accountEnabled" => true
             }
           ]
-    }
+        })
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", users_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     override_endpoint_url("http://localhost:#{bypass.port}/")
@@ -59,14 +59,14 @@ defmodule Domain.Mocks.MicrosoftEntraDirectory do
     bypass
   end
 
-  def mock_groups_list_endpoint(bypass, groups \\ nil) do
+  def mock_groups_list_endpoint(bypass, status, resp \\ nil) do
     groups_list_endpoint_path = "v1.0/groups"
 
-    resp = %{
-      "@odata.context" => "https://graph.microsoft.com/v1.0/$metadata#groups(id,displayName)",
-      "value" =>
-        groups ||
-          [
+    resp =
+      resp ||
+        Jason.encode!(%{
+          "@odata.context" => "https://graph.microsoft.com/v1.0/$metadata#groups(id,displayName)",
+          "value" => [
             %{
               "id" => "962F077E-CAA2-4873-9D7D-A37CD58C06F5",
               "displayName" => "Engineering"
@@ -80,14 +80,14 @@ defmodule Domain.Mocks.MicrosoftEntraDirectory do
               "displayName" => "All"
             }
           ]
-    }
+        })
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", groups_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     override_endpoint_url("http://localhost:#{bypass.port}/")
@@ -95,45 +95,46 @@ defmodule Domain.Mocks.MicrosoftEntraDirectory do
     bypass
   end
 
-  def mock_group_members_list_endpoint(bypass, group_id, members \\ nil) do
+  def mock_group_members_list_endpoint(bypass, group_id, status, resp \\ nil) do
     group_members_list_endpoint_path =
       "v1.0/groups/#{group_id}/transitiveMembers/microsoft.graph.user"
 
     memberships =
-      members ||
-        [
-          %{
-            "id" => "8FBDDD1B-0E73-4CD0-AD38-2ACEA67814EE",
-            "displayName" => "John Doe",
-            "userPrincipalName" => "jdoe@example.local",
-            "accountEnabled" => true
-          },
-          %{
-            "id" => "0B69CEE0-B884-4CAD-B7E3-DDD4D53034FB",
-            "displayName" => "Jane Smith",
-            "userPrincipalName" => "jsmith@example.local",
-            "accountEnabled" => true
-          },
-          %{
-            "id" => "84F44A7C-DC31-4B2B-83F6-6CFCF0AA2456",
-            "displayName" => "Bob Smith",
-            "userPrincipalName" => "bsmith@example.local",
-            "accountEnabled" => true
-          }
-        ]
+      [
+        %{
+          "id" => "8FBDDD1B-0E73-4CD0-AD38-2ACEA67814EE",
+          "displayName" => "John Doe",
+          "userPrincipalName" => "jdoe@example.local",
+          "accountEnabled" => true
+        },
+        %{
+          "id" => "0B69CEE0-B884-4CAD-B7E3-DDD4D53034FB",
+          "displayName" => "Jane Smith",
+          "userPrincipalName" => "jsmith@example.local",
+          "accountEnabled" => true
+        },
+        %{
+          "id" => "84F44A7C-DC31-4B2B-83F6-6CFCF0AA2456",
+          "displayName" => "Bob Smith",
+          "userPrincipalName" => "bsmith@example.local",
+          "accountEnabled" => true
+        }
+      ]
 
-    resp = %{
-      "@odata.context" =>
-        "https://graph.microsoft.com/v1.0/$metadata#users(id,displayName,userPrincipalName,accountEnabled)",
-      "value" => memberships
-    }
+    resp =
+      resp ||
+        Jason.encode!(%{
+          "@odata.context" =>
+            "https://graph.microsoft.com/v1.0/$metadata#users(id,displayName,userPrincipalName,accountEnabled)",
+          "value" => memberships
+        })
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", group_members_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     override_endpoint_url("http://localhost:#{bypass.port}/")

elixir/apps/domain/test/support/mocks/okta_directory.ex

@@ -2,13 +2,13 @@ defmodule Domain.Mocks.OktaDirectory do
   @okta_icon_md "https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-medium.30ce6d4085dff29412984e4c191bc874.png"
   @okta_icon_lg "https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-large.c3cb8cda8ae0add1b4fe928f5844dbe3.png"
 
-  def mock_users_list_endpoint(bypass, users \\ nil) do
+  def mock_users_list_endpoint(bypass, status, resp \\ nil) do
     users_list_endpoint_path = "api/v1/users"
     okta_base_url = "http://localhost:#{bypass.port}"
 
     resp =
-      users ||
-        [
+      resp ||
+        Jason.encode!([
           %{
             "id" => "OT6AZkcmzkDXwkXcjTHY",
             "status" => "ACTIVE",
@@ -57,26 +57,26 @@ defmodule Domain.Mocks.OktaDirectory do
               }
             }
           }
-        ]
+        ])
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", users_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     bypass
   end
 
-  def mock_groups_list_endpoint(bypass, groups \\ nil) do
+  def mock_groups_list_endpoint(bypass, status, resp \\ nil) do
     groups_list_endpoint_path = "api/v1/groups"
     okta_base_url = "http://localhost:#{bypass.port}"
 
     resp =
-      groups ||
-        [
+      resp ||
+        Jason.encode!([
           %{
             "id" => "00gezqhvv4IFj2Avg5d7",
             "created" => "2024-02-07T04:32:03.000Z",
@@ -214,26 +214,26 @@ defmodule Domain.Mocks.OktaDirectory do
               }
             }
           }
-        ]
+        ])
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", groups_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     bypass
   end
 
-  def mock_group_members_list_endpoint(bypass, group_id, members \\ nil) do
+  def mock_group_members_list_endpoint(bypass, group_id, status, resp \\ nil) do
     group_members_list_endpoint_path = "api/v1/groups/#{group_id}/users"
     okta_base_url = "http://localhost:#{bypass.port}"
 
     resp =
-      members ||
-        [
+      resp ||
+        Jason.encode!([
           %{
             "id" => "00ue1rr3zgV1DjyfL5d7",
             "status" => "ACTIVE",
@@ -314,14 +314,14 @@ defmodule Domain.Mocks.OktaDirectory do
               }
             }
           }
-        ]
+        ])
 
     test_pid = self()
 
     Bypass.expect(bypass, "GET", group_members_list_endpoint_path, fn conn ->
       conn = Plug.Conn.fetch_query_params(conn)
       send(test_pid, {:bypass_request, conn})
-      Plug.Conn.send_resp(conn, 200, Jason.encode!(resp))
+      Plug.Conn.send_resp(conn, status, resp)
     end)
 
     bypass