From 28e35b6500702f42aac1c167694f81bbd5c94dcd Mon Sep 17 00:00:00 2001 From: Brian Manifold Date: Wed, 22 Oct 2025 14:13:59 -0700 Subject: [PATCH] refactor(portal): remove all soft delete columns (#10685) Why: * Now that soft delete fields have been removed being referenced in the codebase and soft deleted rows have been removed from the DB we can now remove any remaining traces of soft delete functionality. Resolves #8187 --- elixir/apps/api/lib/api/client/channel.ex | 1 - .../change_logs/replication_connection.ex | 7 - .../lib/domain/changes/hooks/accounts.ex | 17 +-- .../changes/hooks/actor_group_memberships.ex | 6 +- .../lib/domain/changes/hooks/clients.ex | 12 -- .../domain/changes/hooks/gateway_groups.ex | 7 - .../lib/domain/changes/hooks/gateways.ex | 13 +- .../lib/domain/changes/hooks/policies.ex | 19 +-- .../lib/domain/changes/hooks/resources.ex | 12 -- .../domain/lib/domain/changes/hooks/tokens.ex | 22 +--- elixir/apps/domain/lib/domain/flows.ex | 6 +- ...210560_remove_deleted_at_from_accounts.exs | 9 ++ ...20210561_remove_deleted_at_from_actors.exs | 9 ++ ...62_remove_deleted_at_from_actor_groups.exs | 9 ++ ..._remove_deleted_at_from_auth_providers.exs | 9 ++ ...remove_deleted_at_from_auth_identities.exs | 9 ++ ...0210565_remove_deleted_at_from_clients.exs | 9 ++ ...210566_remove_deleted_at_from_gateways.exs | 9 ++ ..._remove_deleted_at_from_gateway_groups.exs | 9 ++ ...210568_remove_deleted_at_from_policies.exs | 9 ++ ...20210569_remove_deleted_at_from_relays.exs | 9 ++ ...70_remove_deleted_at_from_relay_groups.exs | 9 ++ ...10571_remove_deleted_at_from_resources.exs | 9 ++ ...20210572_remove_deleted_at_from_tokens.exs | 9 ++ ...0839_remove_replaced_by_from_resources.exs | 12 ++ ...10840_remove_replaced_by_from_policies.exs | 12 ++ ...27_remove_persistent_id_from_resources.exs | 9 ++ ...228_remove_persistent_id_from_policies.exs | 9 ++ .../replication_connection_test.exs | 121 ------------------ .../domain/changes/hooks/accounts_test.exs | 28 ++-- .../hooks/actor_group_memberships_test.exs | 19 +-- .../domain/changes/hooks/clients_test.exs | 11 -- .../domain/changes/hooks/gateways_test.exs | 11 -- .../domain/changes/hooks/policies_test.exs | 17 --- .../domain/changes/hooks/resources_test.exs | 20 --- .../test/domain/changes/hooks/tokens_test.exs | 17 +-- elixir/apps/domain/test/domain/repo_test.exs | 33 +++-- .../domain/test/support/fixtures/gateways.ex | 1 - .../domain/test/support/fixtures/relays.ex | 1 - .../test/web/live/resources/index_test.exs | 10 +- 40 files changed, 212 insertions(+), 358 deletions(-) create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210560_remove_deleted_at_from_accounts.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210561_remove_deleted_at_from_actors.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210562_remove_deleted_at_from_actor_groups.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210563_remove_deleted_at_from_auth_providers.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210564_remove_deleted_at_from_auth_identities.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210565_remove_deleted_at_from_clients.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210566_remove_deleted_at_from_gateways.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210567_remove_deleted_at_from_gateway_groups.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210568_remove_deleted_at_from_policies.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210569_remove_deleted_at_from_relays.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210570_remove_deleted_at_from_relay_groups.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210571_remove_deleted_at_from_resources.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210572_remove_deleted_at_from_tokens.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210839_remove_replaced_by_from_resources.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020210840_remove_replaced_by_from_policies.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020211227_remove_persistent_id_from_resources.exs create mode 100644 elixir/apps/domain/priv/repo/migrations/20251020211228_remove_persistent_id_from_policies.exs diff --git a/elixir/apps/api/lib/api/client/channel.ex b/elixir/apps/api/lib/api/client/channel.ex index 7fb26bebf..8e2ae5daf 100644 --- a/elixir/apps/api/lib/api/client/channel.ex +++ b/elixir/apps/api/lib/api/client/channel.ex @@ -784,7 +784,6 @@ defmodule API.Client.Channel do %{assigns: %{client: %{id: client_id}}} = socket ) when id == client_id do - # TODO: Hard delete # Deleting a client won't necessary delete its tokens in the case of a headless client. # So we explicitly handle the deleted client here by forcing it to reconnect. {:stop, :shutdown, socket} diff --git a/elixir/apps/domain/lib/domain/change_logs/replication_connection.ex b/elixir/apps/domain/lib/domain/change_logs/replication_connection.ex index 3fc108973..f3704d903 100644 --- a/elixir/apps/domain/lib/domain/change_logs/replication_connection.ex +++ b/elixir/apps/domain/lib/domain/change_logs/replication_connection.ex @@ -5,13 +5,6 @@ defmodule Domain.ChangeLogs.ReplicationConnection do # Bump this to signify a change in the audit log schema. Use with care. @vsn 0 - # Avoid overwhelming the change log with soft-deleted records getting hard-deleted en masse. - # Can be removed after https://github.com/firezone/firezone/issues/8187 is shipped. - def on_write(state, _lsn, :delete, _table, %{"deleted_at" => deleted_at}, _data) - when not is_nil(deleted_at) do - state - end - # Ignore token writes for relay_groups since these are not expected to have an account_id def on_write(state, _lsn, _op, "tokens", %{"type" => "relay_group"}, _data), do: state def on_write(state, _lsn, _op, "tokens", _old_data, %{"type" => "relay_group"}), do: state diff --git a/elixir/apps/domain/lib/domain/changes/hooks/accounts.ex b/elixir/apps/domain/lib/domain/changes/hooks/accounts.ex index f3a396411..3cd1c6cf6 100644 --- a/elixir/apps/domain/lib/domain/changes/hooks/accounts.ex +++ b/elixir/apps/domain/lib/domain/changes/hooks/accounts.ex @@ -18,16 +18,11 @@ defmodule Domain.Changes.Hooks.Accounts do %{"disabled_at" => disabled_at} ) when not is_nil(disabled_at) do - on_delete(lsn, old_data) - end + # TODO: Potentially revisit whether this should be handled here + # or handled closer to where the PubSub message is received. + account = struct_from_params(Accounts.Account, old_data) + Flows.delete_flows_for(account) - # Account soft-deleted - process as a delete - def on_update( - lsn, - %{"deleted_at" => nil} = old_data, - %{"deleted_at" => deleted_at} - ) - when not is_nil(deleted_at) do on_delete(lsn, old_data) end @@ -45,10 +40,6 @@ defmodule Domain.Changes.Hooks.Accounts do account = struct_from_params(Accounts.Account, old_data) change = %Change{lsn: lsn, op: :delete, old_struct: account} - # TODO: Hard delete - # This can be removed upon implementation of hard delete - Flows.delete_flows_for(account) - PubSub.Account.broadcast(account.id, change) end end diff --git a/elixir/apps/domain/lib/domain/changes/hooks/actor_group_memberships.ex b/elixir/apps/domain/lib/domain/changes/hooks/actor_group_memberships.ex index 17043c698..c7848f287 100644 --- a/elixir/apps/domain/lib/domain/changes/hooks/actor_group_memberships.ex +++ b/elixir/apps/domain/lib/domain/changes/hooks/actor_group_memberships.ex @@ -1,6 +1,6 @@ defmodule Domain.Changes.Hooks.ActorGroupMemberships do @behaviour Domain.Changes.Hooks - alias Domain.{Actors, Changes.Change, Flows, PubSub} + alias Domain.{Actors, Changes.Change, PubSub} import Domain.SchemaHelpers @impl true @@ -19,10 +19,6 @@ defmodule Domain.Changes.Hooks.ActorGroupMemberships do membership = struct_from_params(Actors.Membership, old_data) change = %Change{lsn: lsn, op: :delete, old_struct: membership} - # TODO: Hard delete - # This can be removed upon implementation of hard delete - Flows.delete_flows_for(membership) - PubSub.Account.broadcast(membership.account_id, change) end end diff --git a/elixir/apps/domain/lib/domain/changes/hooks/clients.ex b/elixir/apps/domain/lib/domain/changes/hooks/clients.ex index 830bc652e..507916cb8 100644 --- a/elixir/apps/domain/lib/domain/changes/hooks/clients.ex +++ b/elixir/apps/domain/lib/domain/changes/hooks/clients.ex @@ -7,14 +7,6 @@ defmodule Domain.Changes.Hooks.Clients do def on_insert(_lsn, _data), do: :ok @impl true - - # Soft-delete - def on_update(lsn, %{"deleted_at" => nil} = old_data, %{"deleted_at" => deleted_at}) - when not is_nil(deleted_at) do - on_delete(lsn, old_data) - end - - # Regular update def on_update(lsn, old_data, data) do old_client = struct_from_params(Clients.Client, old_data) client = struct_from_params(Clients.Client, data) @@ -35,10 +27,6 @@ defmodule Domain.Changes.Hooks.Clients do client = struct_from_params(Clients.Client, old_data) change = %Change{lsn: lsn, op: :delete, old_struct: client} - # TODO: Hard delete - # This can be removed upon implementation of hard delete - Flows.delete_flows_for(client) - PubSub.Account.broadcast(client.account_id, change) end end diff --git a/elixir/apps/domain/lib/domain/changes/hooks/gateway_groups.ex b/elixir/apps/domain/lib/domain/changes/hooks/gateway_groups.ex index cbd6af200..02f56c627 100644 --- a/elixir/apps/domain/lib/domain/changes/hooks/gateway_groups.ex +++ b/elixir/apps/domain/lib/domain/changes/hooks/gateway_groups.ex @@ -6,14 +6,7 @@ defmodule Domain.Changes.Hooks.GatewayGroups do @impl true def on_insert(_lsn, _data), do: :ok - # Soft-delete @impl true - def on_update(lsn, %{"deleted_at" => nil} = old_data, %{"deleted_at" => deleted_at}) - when not is_nil(deleted_at) do - on_delete(lsn, old_data) - end - - # Regular update def on_update(lsn, old_data, data) do old_gateway_group = struct_from_params(Gateways.Group, old_data) gateway_group = struct_from_params(Gateways.Group, data) diff --git a/elixir/apps/domain/lib/domain/changes/hooks/gateways.ex b/elixir/apps/domain/lib/domain/changes/hooks/gateways.ex index 0fee631d4..20e1b1fec 100644 --- a/elixir/apps/domain/lib/domain/changes/hooks/gateways.ex +++ b/elixir/apps/domain/lib/domain/changes/hooks/gateways.ex @@ -1,19 +1,12 @@ defmodule Domain.Changes.Hooks.Gateways do @behaviour Domain.Changes.Hooks - alias Domain.{Changes.Change, Flows, Gateways, PubSub} + alias Domain.{Changes.Change, Gateways, PubSub} import Domain.SchemaHelpers @impl true def on_insert(_lsn, _data), do: :ok - # Soft-delete @impl true - def on_update(lsn, %{"deleted_at" => nil} = old_data, %{"deleted_at" => deleted_at}) - when not is_nil(deleted_at) do - on_delete(lsn, old_data) - end - - # Regular update def on_update(_lsn, _old_data, _data), do: :ok @impl true @@ -21,10 +14,6 @@ defmodule Domain.Changes.Hooks.Gateways do gateway = struct_from_params(Gateways.Gateway, old_data) change = %Change{lsn: lsn, op: :delete, old_struct: gateway} - # TODO: Hard delete - # This can be removed upon implementation of hard delete - Flows.delete_flows_for(gateway) - PubSub.Account.broadcast(gateway.account_id, change) end end diff --git a/elixir/apps/domain/lib/domain/changes/hooks/policies.ex b/elixir/apps/domain/lib/domain/changes/hooks/policies.ex index 511ba8d05..80e91f30b 100644 --- a/elixir/apps/domain/lib/domain/changes/hooks/policies.ex +++ b/elixir/apps/domain/lib/domain/changes/hooks/policies.ex @@ -14,9 +14,14 @@ defmodule Domain.Changes.Hooks.Policies do @impl true # Disable - process as delete - def on_update(lsn, %{"disabled_at" => nil}, %{"disabled_at" => disabled_at} = data) + def on_update(lsn, %{"disabled_at" => nil} = old_data, %{"disabled_at" => disabled_at}) when not is_nil(disabled_at) do - on_delete(lsn, data) + # TODO: Potentially revisit whether this should be handled here + # or handled closer to where the PubSub message is received. + policy = struct_from_params(Policies.Policy, old_data) + Flows.delete_flows_for(policy) + + on_delete(lsn, old_data) end # Enable - process as insert @@ -25,12 +30,6 @@ defmodule Domain.Changes.Hooks.Policies do on_insert(lsn, data) end - # Soft-delete - process as delete - def on_update(lsn, %{"deleted_at" => nil} = old_data, %{"deleted_at" => deleted_at}) - when not is_nil(deleted_at) do - on_delete(lsn, old_data) - end - # Regular update def on_update(lsn, old_data, data) do old_policy = struct_from_params(Policies.Policy, old_data) @@ -55,10 +54,6 @@ defmodule Domain.Changes.Hooks.Policies do policy = struct_from_params(Policies.Policy, old_data) change = %Change{lsn: lsn, op: :delete, old_struct: policy} - # TODO: Hard delete - # This can be removed upon implementation of hard delete - Flows.delete_flows_for(policy) - PubSub.Account.broadcast(policy.account_id, change) end end diff --git a/elixir/apps/domain/lib/domain/changes/hooks/resources.ex b/elixir/apps/domain/lib/domain/changes/hooks/resources.ex index dd3fd4c2f..4075152e4 100644 --- a/elixir/apps/domain/lib/domain/changes/hooks/resources.ex +++ b/elixir/apps/domain/lib/domain/changes/hooks/resources.ex @@ -12,14 +12,6 @@ defmodule Domain.Changes.Hooks.Resources do end @impl true - - # Soft-delete - process as delete - def on_update(lsn, %{"deleted_at" => nil} = old_data, %{"deleted_at" => deleted_at}) - when not is_nil(deleted_at) do - on_delete(lsn, old_data) - end - - # Regular update def on_update(lsn, old_data, data) do old_resource = struct_from_params(Resources.Resource, old_data) resource = struct_from_params(Resources.Resource, data) @@ -48,10 +40,6 @@ defmodule Domain.Changes.Hooks.Resources do resource = struct_from_params(Resources.Resource, old_data) change = %Change{lsn: lsn, op: :delete, old_struct: resource} - # TODO: Hard delete - # This can be removed upon implementation of hard delete - Flows.delete_flows_for(resource) - PubSub.Account.broadcast(resource.account_id, change) end end diff --git a/elixir/apps/domain/lib/domain/changes/hooks/tokens.ex b/elixir/apps/domain/lib/domain/changes/hooks/tokens.ex index a9c3d3128..bd52983a0 100644 --- a/elixir/apps/domain/lib/domain/changes/hooks/tokens.ex +++ b/elixir/apps/domain/lib/domain/changes/hooks/tokens.ex @@ -1,39 +1,19 @@ defmodule Domain.Changes.Hooks.Tokens do @behaviour Domain.Changes.Hooks - alias Domain.{Flows, PubSub, Tokens} + alias Domain.{PubSub, Tokens} import Domain.SchemaHelpers @impl true def on_insert(_lsn, _data), do: :ok @impl true - # updates for email and relay_group tokens have no side effects - def on_update(_lsn, %{"type" => "email"}, _data), do: :ok - - def on_update(_lsn, _old_data, %{"type" => "email"}), do: :ok - - def on_update(_lsn, %{"type" => "relay_group"}, _data), do: :ok - - def on_update(_lsn, _old_data, %{"type" => "relay_group"}), do: :ok - - # Soft-delete - process as delete - def on_update(lsn, %{"deleted_at" => nil} = old_data, %{"deleted_at" => deleted_at}) - when not is_nil(deleted_at) do - on_delete(lsn, old_data) - end - - # Regular update def on_update(_lsn, _old_data, _new_data), do: :ok @impl true def on_delete(_lsn, old_data) do token = struct_from_params(Tokens.Token, old_data) - # TODO: Hard delete - # This can be removed upon implementation of hard delete - Flows.delete_flows_for(token) - # We don't need to broadcast deleted tokens since the disconnect_socket/1 # function will handle any disconnects for us directly. diff --git a/elixir/apps/domain/lib/domain/flows.ex b/elixir/apps/domain/lib/domain/flows.ex index f0d869a49..97ce59076 100644 --- a/elixir/apps/domain/lib/domain/flows.ex +++ b/elixir/apps/domain/lib/domain/flows.ex @@ -1,6 +1,6 @@ defmodule Domain.Flows do alias Domain.Repo - alias Domain.{Auth, Actors, Clients, Gateways, Resources, Policies, Tokens} + alias Domain.{Auth, Actors, Clients, Gateways, Resources, Policies} alias Domain.Flows.{Authorizer, Flow} require Ecto.Query require Logger @@ -75,10 +75,6 @@ defmodule Domain.Flows do # since we won't need to be so careful about reject_access messages to the gateway. def reauthorize_flow(%Flow{} = flow) do with {:ok, client} <- Clients.fetch_client_by_id(flow.client_id, preload: :identity), - # TODO: Hard delete - # We need to ensure token and gateway haven't been deleted after the initial flow was created - # This can be removed after hard-delete since we'll get a DB error if these associations no longer exist - {:ok, _token} <- Tokens.fetch_token_by_id(flow.token_id), {:ok, gateway} <- Gateways.fetch_gateway_by_id(flow.gateway_id), # We only want to reauthorize the resource for this gateway if the resource is still connected to its # gateway_group. diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210560_remove_deleted_at_from_accounts.exs b/elixir/apps/domain/priv/repo/migrations/20251020210560_remove_deleted_at_from_accounts.exs new file mode 100644 index 000000000..dee8aac43 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210560_remove_deleted_at_from_accounts.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromAccounts do + use Ecto.Migration + + def change do + alter table(:accounts) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210561_remove_deleted_at_from_actors.exs b/elixir/apps/domain/priv/repo/migrations/20251020210561_remove_deleted_at_from_actors.exs new file mode 100644 index 000000000..cde6f7b67 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210561_remove_deleted_at_from_actors.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromActors do + use Ecto.Migration + + def change do + alter table(:actors) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210562_remove_deleted_at_from_actor_groups.exs b/elixir/apps/domain/priv/repo/migrations/20251020210562_remove_deleted_at_from_actor_groups.exs new file mode 100644 index 000000000..fb76404d0 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210562_remove_deleted_at_from_actor_groups.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromActorGroups do + use Ecto.Migration + + def change do + alter table(:actor_groups) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210563_remove_deleted_at_from_auth_providers.exs b/elixir/apps/domain/priv/repo/migrations/20251020210563_remove_deleted_at_from_auth_providers.exs new file mode 100644 index 000000000..738e77f8b --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210563_remove_deleted_at_from_auth_providers.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromAuthProviders do + use Ecto.Migration + + def change do + alter table(:auth_providers) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210564_remove_deleted_at_from_auth_identities.exs b/elixir/apps/domain/priv/repo/migrations/20251020210564_remove_deleted_at_from_auth_identities.exs new file mode 100644 index 000000000..41364b84f --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210564_remove_deleted_at_from_auth_identities.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromAuthIdentities do + use Ecto.Migration + + def change do + alter table(:auth_identities) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210565_remove_deleted_at_from_clients.exs b/elixir/apps/domain/priv/repo/migrations/20251020210565_remove_deleted_at_from_clients.exs new file mode 100644 index 000000000..711b9e161 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210565_remove_deleted_at_from_clients.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromClients do + use Ecto.Migration + + def change do + alter table(:clients) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210566_remove_deleted_at_from_gateways.exs b/elixir/apps/domain/priv/repo/migrations/20251020210566_remove_deleted_at_from_gateways.exs new file mode 100644 index 000000000..6c9bfa431 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210566_remove_deleted_at_from_gateways.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromGateways do + use Ecto.Migration + + def change do + alter table(:gateways) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210567_remove_deleted_at_from_gateway_groups.exs b/elixir/apps/domain/priv/repo/migrations/20251020210567_remove_deleted_at_from_gateway_groups.exs new file mode 100644 index 000000000..6fc50be0b --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210567_remove_deleted_at_from_gateway_groups.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromGatewayGroups do + use Ecto.Migration + + def change do + alter table(:gateway_groups) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210568_remove_deleted_at_from_policies.exs b/elixir/apps/domain/priv/repo/migrations/20251020210568_remove_deleted_at_from_policies.exs new file mode 100644 index 000000000..1b85ed716 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210568_remove_deleted_at_from_policies.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromPolicies do + use Ecto.Migration + + def change do + alter table(:policies) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210569_remove_deleted_at_from_relays.exs b/elixir/apps/domain/priv/repo/migrations/20251020210569_remove_deleted_at_from_relays.exs new file mode 100644 index 000000000..0b3c5b1bb --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210569_remove_deleted_at_from_relays.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromRelays do + use Ecto.Migration + + def change do + alter table(:relays) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210570_remove_deleted_at_from_relay_groups.exs b/elixir/apps/domain/priv/repo/migrations/20251020210570_remove_deleted_at_from_relay_groups.exs new file mode 100644 index 000000000..75e546336 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210570_remove_deleted_at_from_relay_groups.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromRelayGroups do + use Ecto.Migration + + def change do + alter table(:relay_groups) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210571_remove_deleted_at_from_resources.exs b/elixir/apps/domain/priv/repo/migrations/20251020210571_remove_deleted_at_from_resources.exs new file mode 100644 index 000000000..70b096ab7 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210571_remove_deleted_at_from_resources.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromResources do + use Ecto.Migration + + def change do + alter table(:resources) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210572_remove_deleted_at_from_tokens.exs b/elixir/apps/domain/priv/repo/migrations/20251020210572_remove_deleted_at_from_tokens.exs new file mode 100644 index 000000000..ed7f520ea --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210572_remove_deleted_at_from_tokens.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemoveDeletedAtFromTokens do + use Ecto.Migration + + def change do + alter table(:tokens) do + remove(:deleted_at, :utc_datetime_usec) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210839_remove_replaced_by_from_resources.exs b/elixir/apps/domain/priv/repo/migrations/20251020210839_remove_replaced_by_from_resources.exs new file mode 100644 index 000000000..9528c6a21 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210839_remove_replaced_by_from_resources.exs @@ -0,0 +1,12 @@ +defmodule Domain.Repo.Migrations.RemoveReplacedByFromResources do + use Ecto.Migration + + def change do + alter table(:resources) do + remove( + :replaced_by_resource_id, + references(:resources, type: :binary_id, on_delete: :nilify_all) + ) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020210840_remove_replaced_by_from_policies.exs b/elixir/apps/domain/priv/repo/migrations/20251020210840_remove_replaced_by_from_policies.exs new file mode 100644 index 000000000..0d4072e6c --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020210840_remove_replaced_by_from_policies.exs @@ -0,0 +1,12 @@ +defmodule Domain.Repo.Migrations.RemoveReplacedByFromPolicies do + use Ecto.Migration + + def change do + alter table(:policies) do + remove( + :replaced_by_policy_id, + references(:policies, type: :binary_id, on_delete: :nilify_all) + ) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020211227_remove_persistent_id_from_resources.exs b/elixir/apps/domain/priv/repo/migrations/20251020211227_remove_persistent_id_from_resources.exs new file mode 100644 index 000000000..914be3f6b --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020211227_remove_persistent_id_from_resources.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemovePersistentIdFromResources do + use Ecto.Migration + + def change do + alter table(:resources) do + remove(:persistent_id, :uuid) + end + end +end diff --git a/elixir/apps/domain/priv/repo/migrations/20251020211228_remove_persistent_id_from_policies.exs b/elixir/apps/domain/priv/repo/migrations/20251020211228_remove_persistent_id_from_policies.exs new file mode 100644 index 000000000..0573afe17 --- /dev/null +++ b/elixir/apps/domain/priv/repo/migrations/20251020211228_remove_persistent_id_from_policies.exs @@ -0,0 +1,9 @@ +defmodule Domain.Repo.Migrations.RemovePersistentIdFromPolicies do + use Ecto.Migration + + def change do + alter table(:policies) do + remove(:persistent_id, :uuid) + end + end +end diff --git a/elixir/apps/domain/test/domain/change_logs/replication_connection_test.exs b/elixir/apps/domain/test/domain/change_logs/replication_connection_test.exs index 79b2b1bc9..27b918dc7 100644 --- a/elixir/apps/domain/test/domain/change_logs/replication_connection_test.exs +++ b/elixir/apps/domain/test/domain/change_logs/replication_connection_test.exs @@ -307,60 +307,6 @@ defmodule Domain.ChangeLogs.ReplicationConnectionTest do assert attrs.account_id == account.id assert attrs.vsn == 0 end - - test "ignores soft-deleted records", %{account: account} do - table = "resources" - - old_data = %{ - "id" => Ecto.UUID.generate(), - "account_id" => account.id, - "name" => "soft deleted", - "deleted_at" => "2024-01-01T00:00:00Z" - } - - lsn = 12348 - - initial_state = %{flush_buffer: %{}} - - result_state = - ReplicationConnection.on_write( - initial_state, - lsn, - :delete, - table, - old_data, - nil - ) - - # Buffer should remain unchanged - assert map_size(result_state.flush_buffer) == 0 - assert result_state == initial_state - end - - test "processes record without deleted_at field", %{account: account} do - old_data = %{ - "id" => Ecto.UUID.generate(), - "account_id" => account.id, - "name" => "no deleted_at field" - } - - state = %{flush_buffer: %{}} - lsn = 400 - - result_state = - ReplicationConnection.on_write( - state, - lsn, - :delete, - "resources", - old_data, - nil - ) - - assert map_size(result_state.flush_buffer) == 1 - attrs = result_state.flush_buffer[lsn] - assert attrs.op == :delete - end end describe "multiple operations and buffer accumulation" do @@ -417,73 +363,6 @@ defmodule Domain.ChangeLogs.ReplicationConnectionTest do assert state3.flush_buffer[101].op == :update assert state3.flush_buffer[102].op == :delete end - - test "mixed operations with soft deletes", %{account: account} do - state = %{flush_buffer: %{}} - - # Regular insert - state1 = - ReplicationConnection.on_write( - state, - 100, - :insert, - "resources", - nil, - %{"id" => Ecto.UUID.generate(), "account_id" => account.id, "name" => "test"} - ) - - # Regular update - resource_id = Ecto.UUID.generate() - - state2 = - ReplicationConnection.on_write( - state1, - 101, - :update, - "resources", - %{"id" => resource_id, "account_id" => account.id, "name" => "test"}, - %{"id" => resource_id, "account_id" => account.id, "name" => "updated"} - ) - - # Soft delete (should be ignored) - state3 = - ReplicationConnection.on_write( - state2, - 102, - :delete, - "resources", - %{ - "id" => resource_id, - "account_id" => account.id, - "name" => "updated", - "deleted_at" => "2024-01-01T00:00:00Z" - }, - nil - ) - - # Hard delete (should be included) - state4 = - ReplicationConnection.on_write( - state3, - 103, - :delete, - "resources", - %{ - "id" => resource_id, - "account_id" => account.id, - "name" => "updated", - "deleted_at" => nil - }, - nil - ) - - # Should have 3 operations: insert, update, hard delete (soft delete ignored) - assert map_size(state4.flush_buffer) == 3 - assert Map.has_key?(state4.flush_buffer, 100) - assert Map.has_key?(state4.flush_buffer, 101) - refute Map.has_key?(state4.flush_buffer, 102) - assert Map.has_key?(state4.flush_buffer, 103) - end end describe "on_flush/1" do diff --git a/elixir/apps/domain/test/domain/changes/hooks/accounts_test.exs b/elixir/apps/domain/test/domain/changes/hooks/accounts_test.exs index 5ac252eae..8a442cbbf 100644 --- a/elixir/apps/domain/test/domain/changes/hooks/accounts_test.exs +++ b/elixir/apps/domain/test/domain/changes/hooks/accounts_test.exs @@ -30,6 +30,24 @@ defmodule Domain.Changes.Hooks.AccountsTest do assert account.id == account_id end + + test "deletes associated flows when account is disabled" do + account = Fixtures.Accounts.create_account() + flow = Fixtures.Flows.create_flow(account: account) + + old_data = %{ + "id" => account.id, + "disabled_at" => nil + } + + data = %{ + "id" => account.id, + "disabled_at" => "2023-10-01T00:00:00Z" + } + + assert :ok == on_update(0, old_data, data) + assert Repo.get_by(Domain.Flows.Flow, id: flow.id) == nil + end end describe "delete/1" do @@ -43,15 +61,5 @@ defmodule Domain.Changes.Hooks.AccountsTest do assert_receive %Change{op: :delete, old_struct: %Accounts.Account{} = account, lsn: 0} assert account.id == account_id end - - test "deletes associated flows when account is deleted" do - account = Fixtures.Accounts.create_account() - flow = Fixtures.Flows.create_flow(account: account) - - old_data = %{"id" => account.id} - - assert :ok == on_delete(0, old_data) - assert Repo.get_by(Domain.Flows.Flow, id: flow.id) == nil - end end end diff --git a/elixir/apps/domain/test/domain/changes/hooks/actor_group_memberships_test.exs b/elixir/apps/domain/test/domain/changes/hooks/actor_group_memberships_test.exs index f9e554b8a..c5c071e46 100644 --- a/elixir/apps/domain/test/domain/changes/hooks/actor_group_memberships_test.exs +++ b/elixir/apps/domain/test/domain/changes/hooks/actor_group_memberships_test.exs @@ -1,7 +1,7 @@ defmodule Domain.Changes.Hooks.ActorGroupMembershipsTest do use Domain.DataCase, async: true import Domain.Changes.Hooks.ActorGroupMemberships - alias Domain.{Actors, Changes.Change, Flows, PubSub} + alias Domain.{Actors, Changes.Change, PubSub} describe "insert/1" do test "broadcasts membership" do @@ -67,22 +67,5 @@ defmodule Domain.Changes.Hooks.ActorGroupMembershipsTest do assert membership.actor_id == "00000000-0000-0000-0000-000000000002" assert membership.group_id == "00000000-0000-0000-0000-000000000003" end - - test "deletes flows for membership", %{account: account, membership: membership} do - flow = Fixtures.Flows.create_flow(account: account, actor_group_membership: membership) - unrelated_flow = Fixtures.Flows.create_flow(account: account) - - old_data = %{ - "id" => membership.id, - "account_id" => membership.account_id, - "actor_id" => membership.actor_id, - "group_id" => membership.group_id - } - - assert ^flow = Repo.get_by(Flows.Flow, actor_group_membership_id: membership.id) - assert :ok == on_delete(0, old_data) - assert nil == Repo.get_by(Flows.Flow, actor_group_membership_id: membership.id) - assert ^unrelated_flow = Repo.get_by(Flows.Flow, id: unrelated_flow.id) - end end end diff --git a/elixir/apps/domain/test/domain/changes/hooks/clients_test.exs b/elixir/apps/domain/test/domain/changes/hooks/clients_test.exs index 9a8598b17..fa589aea4 100644 --- a/elixir/apps/domain/test/domain/changes/hooks/clients_test.exs +++ b/elixir/apps/domain/test/domain/changes/hooks/clients_test.exs @@ -73,16 +73,5 @@ defmodule Domain.Changes.Hooks.ClientsTest do assert_receive %Change{op: :delete, old_struct: %Clients.Client{} = deleted_client, lsn: 0} assert deleted_client.id == client.id end - - test "deletes associated flows" do - account = Fixtures.Accounts.create_account() - client = Fixtures.Clients.create_client(account: account) - - old_data = %{"id" => client.id, "account_id" => client.account_id} - - assert flow = Fixtures.Flows.create_flow(client: client, account: account) - assert :ok == on_delete(0, old_data) - refute Repo.get_by(Flows.Flow, id: flow.id) - end end end diff --git a/elixir/apps/domain/test/domain/changes/hooks/gateways_test.exs b/elixir/apps/domain/test/domain/changes/hooks/gateways_test.exs index 0fd5c6622..a66de6c4a 100644 --- a/elixir/apps/domain/test/domain/changes/hooks/gateways_test.exs +++ b/elixir/apps/domain/test/domain/changes/hooks/gateways_test.exs @@ -38,16 +38,5 @@ defmodule Domain.Changes.Hooks.GatewaysTest do assert deleted_gateway.id == gateway.id end - - test "deletes flows" do - account = Fixtures.Accounts.create_account() - gateway = Fixtures.Gateways.create_gateway(account: account) - - old_data = %{"id" => gateway.id, "account_id" => account.id} - - assert flow = Fixtures.Flows.create_flow(gateway: gateway, account: account) - assert :ok = on_delete(0, old_data) - refute Repo.get_by(Domain.Flows.Flow, id: flow.id) - end end end diff --git a/elixir/apps/domain/test/domain/changes/hooks/policies_test.exs b/elixir/apps/domain/test/domain/changes/hooks/policies_test.exs index 448fecfd1..4d4e6aad7 100644 --- a/elixir/apps/domain/test/domain/changes/hooks/policies_test.exs +++ b/elixir/apps/domain/test/domain/changes/hooks/policies_test.exs @@ -212,22 +212,5 @@ defmodule Domain.Changes.Hooks.PoliciesTest do assert policy.actor_group_id == old_data["actor_group_id"] assert policy.resource_id == old_data["resource_id"] end - - test "deletes flows" do - account = Fixtures.Accounts.create_account() - policy = Fixtures.Policies.create_policy(account: account) - - old_data = %{ - "id" => policy.id, - "account_id" => account.id, - "actor_group_id" => policy.actor_group_id, - "resource_id" => policy.resource_id - } - - assert flow = Fixtures.Flows.create_flow(policy: policy, account: account) - - assert :ok = on_delete(0, old_data) - refute Repo.get_by(Domain.Flows.Flow, id: flow.id) - end end end diff --git a/elixir/apps/domain/test/domain/changes/hooks/resources_test.exs b/elixir/apps/domain/test/domain/changes/hooks/resources_test.exs index cfa86480f..7a1028274 100644 --- a/elixir/apps/domain/test/domain/changes/hooks/resources_test.exs +++ b/elixir/apps/domain/test/domain/changes/hooks/resources_test.exs @@ -134,25 +134,5 @@ defmodule Domain.Changes.Hooks.ResourcesTest do assert deleted_resource.ip_stack == resource.ip_stack assert deleted_resource.address_description == resource.address_description end - - test "deletes flows" do - account = Fixtures.Accounts.create_account() - filters = [%{"protocol" => "tcp", "ports" => ["80", "443"]}] - resource = Fixtures.Resources.create_resource(account: account, filters: filters) - - old_data = %{ - "id" => resource.id, - "account_id" => account.id, - "address_description" => resource.address_description, - "type" => resource.type, - "address" => resource.address, - "filters" => filters, - "ip_stack" => resource.ip_stack - } - - assert flow = Fixtures.Flows.create_flow(resource: resource, account: account) - assert :ok = on_delete(0, old_data) - refute Repo.get_by(Flows.Flow, id: flow.id) - end end end diff --git a/elixir/apps/domain/test/domain/changes/hooks/tokens_test.exs b/elixir/apps/domain/test/domain/changes/hooks/tokens_test.exs index b563e2713..a932584a1 100644 --- a/elixir/apps/domain/test/domain/changes/hooks/tokens_test.exs +++ b/elixir/apps/domain/test/domain/changes/hooks/tokens_test.exs @@ -1,7 +1,7 @@ defmodule Domain.Changes.Hooks.TokensTest do use Domain.DataCase, async: true import Domain.Changes.Hooks.Tokens - alias Domain.{Flows, PubSub} + alias Domain.PubSub describe "insert/1" do test "returns :ok" do @@ -41,20 +41,5 @@ defmodule Domain.Changes.Hooks.TokensTest do assert topic == "sessions:#{token.id}" end - - test "deletes flows" do - account = Fixtures.Accounts.create_account() - token = Fixtures.Tokens.create_token(account: account) - - old_data = %{ - "id" => token.id, - "account_id" => account.id, - "type" => token.type - } - - assert flow = Fixtures.Flows.create_flow(account: account, token: token) - assert :ok = on_delete(0, old_data) - refute Repo.get_by(Flows.Flow, id: flow.id) - end end end diff --git a/elixir/apps/domain/test/domain/repo_test.exs b/elixir/apps/domain/test/domain/repo_test.exs index a710f0d7a..19702226c 100644 --- a/elixir/apps/domain/test/domain/repo_test.exs +++ b/elixir/apps/domain/test/domain/repo_test.exs @@ -269,27 +269,26 @@ defmodule Domain.RepoTest do end end - # TODO: BRIAN - Check if this test can be updated to use something other than deleted_at - # test "allows to set custom order", %{ - # account: account, - # query_module: query_module, - # queryable: queryable - # } do - # dt1 = ~U[2000-01-01 00:00:00.000000Z] - # dt2 = ~U[2000-01-02 00:00:00.000000Z] + test "allows to set custom order", %{ + account: account, + query_module: query_module, + queryable: queryable + } do + t1 = ~U[2000-01-01 00:00:00.000000Z] + t2 = ~U[2000-01-02 00:00:00.000000Z] - # Fixtures.Actors.create_actor(account: account) - # |> Fixtures.Actors.update(deleted_at: dt1) + Fixtures.Actors.create_actor(account: account) + |> Fixtures.Actors.update(disabled_at: t1) - # Fixtures.Actors.create_actor(account: account) - # |> Fixtures.Actors.update(deleted_at: dt2) + Fixtures.Actors.create_actor(account: account) + |> Fixtures.Actors.update(disabled_at: t2) - # assert {:ok, [%{deleted_at: ^dt1}, %{deleted_at: ^dt2}], _metadata} = - # list(queryable, query_module, order_by: [{:actors, :asc, :deleted_at}]) + assert {:ok, [%{disabled_at: ^t1}, %{disabled_at: ^t2}], _metadata} = + list(queryable, query_module, order_by: [{:actors, :asc, :disabled_at}]) - # assert {:ok, [%{deleted_at: ^dt2}, %{deleted_at: ^dt1}], _metadata} = - # list(queryable, query_module, order_by: [{:actors, :desc, :deleted_at}]) - # end + assert {:ok, [%{disabled_at: ^t2}, %{disabled_at: ^t1}], _metadata} = + list(queryable, query_module, order_by: [{:actors, :desc, :disabled_at}]) + end test "allows to filter results" do query_module = Domain.Accounts.Account.Query diff --git a/elixir/apps/domain/test/support/fixtures/gateways.ex b/elixir/apps/domain/test/support/fixtures/gateways.ex index 71ea49e10..7daca1bdd 100644 --- a/elixir/apps/domain/test/support/fixtures/gateways.ex +++ b/elixir/apps/domain/test/support/fixtures/gateways.ex @@ -104,7 +104,6 @@ defmodule Domain.Fixtures.Gateways do |> create_group() end) - # TODO: BRIAN - This can likely be removed {_token, attrs} = pop_assoc_fixture(attrs, :token, fn assoc_attrs -> assoc_attrs diff --git a/elixir/apps/domain/test/support/fixtures/relays.ex b/elixir/apps/domain/test/support/fixtures/relays.ex index 9a93db6db..7e5b1f80a 100644 --- a/elixir/apps/domain/test/support/fixtures/relays.ex +++ b/elixir/apps/domain/test/support/fixtures/relays.ex @@ -112,7 +112,6 @@ defmodule Domain.Fixtures.Relays do |> create_group() end) - # TODO: BRIAN - This can likely be removed {_token, attrs} = pop_assoc_fixture(attrs, :token, fn assoc_attrs -> if group.account_id do diff --git a/elixir/apps/web/test/web/live/resources/index_test.exs b/elixir/apps/web/test/web/live/resources/index_test.exs index a6dfb4918..bb21c2559 100644 --- a/elixir/apps/web/test/web/live/resources/index_test.exs +++ b/elixir/apps/web/test/web/live/resources/index_test.exs @@ -310,17 +310,11 @@ defmodule Web.Live.Resources.IndexTest do Domain.Resources.delete_resource(resource, subject) - Changes.Hooks.Resources.on_update( + Changes.Hooks.Resources.on_delete( 0, %{ "id" => resource.id, - "account_id" => account.id, - "deleted_at" => nil - }, - %{ - "id" => resource.id, - "account_id" => account.id, - "deleted_at" => DateTime.utc_now() + "account_id" => account.id } )