Clean views without object metadata (#7153)

Add command for cleaning + clean on object deletion

packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-backfill-workspace-favorites.command.ts

@@ -2,13 +2,14 @@ import { InjectRepository } from '@nestjs/typeorm';
 
 import chalk from 'chalk';
 import { Command } from 'nest-commander';
-import { Repository } from 'typeorm';
+import { In, Repository } from 'typeorm';
 
 import {
   ActiveWorkspacesCommandOptions,
   ActiveWorkspacesCommandRunner,
 } from 'src/database/commands/active-workspaces.command';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
 import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity';
 import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity';
@@ -21,6 +22,8 @@ export class BackfillWorkspaceFavoritesCommand extends ActiveWorkspacesCommandRu
   constructor(
     @InjectRepository(Workspace, 'core')
     protected readonly workspaceRepository: Repository<Workspace>,
+    @InjectRepository(ObjectMetadataEntity, 'metadata')
+    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
     private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
   ) {
     super(workspaceRepository);
@@ -37,11 +40,17 @@ export class BackfillWorkspaceFavoritesCommand extends ActiveWorkspacesCommandRu
       this.logger.log(`Running command for workspace ${workspaceId}`);
 
       try {
-        const workspaceIndexViews = await this.getIndexViews(workspaceId);
+        const allWorkspaceIndexViews = await this.getIndexViews(workspaceId);
+
+        const activeWorkspaceIndexViews =
+          await this.filterViewsWithoutObjectMetadata(
+            workspaceId,
+            allWorkspaceIndexViews,
+          );
 
         await this.createViewWorkspaceFavorites(
           workspaceId,
-          workspaceIndexViews.map((view) => view.id),
+          activeWorkspaceIndexViews.map((view) => view.id),
         );
 
         this.logger.log(
@@ -85,6 +94,26 @@ export class BackfillWorkspaceFavoritesCommand extends ActiveWorkspacesCommandRu
     });
   }
 
+  private async filterViewsWithoutObjectMetadata(
+    workspaceId: string,
+    views: ViewWorkspaceEntity[],
+  ): Promise<ViewWorkspaceEntity[]> {
+    const viewObjectMetadataIds = views.map((view) => view.objectMetadataId);
+
+    const objectMetadataEntities = await this.objectMetadataRepository.find({
+      where: {
+        workspaceId,
+        id: In(viewObjectMetadataIds),
+      },
+    });
+
+    const objectMetadataIds = new Set(
+      objectMetadataEntities.map((entity) => entity.id),
+    );
+
+    return views.filter((view) => objectMetadataIds.has(view.objectMetadataId));
+  }
+
   private async createViewWorkspaceFavorites(
     workspaceId: string,
     viewIds: string[],

packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-clean-views-with-deleted-object-metadata.command.ts

@@ -0,0 +1,115 @@
+import { InjectRepository } from '@nestjs/typeorm';
+
+import chalk from 'chalk';
+import { Command } from 'nest-commander';
+import { In, Repository } from 'typeorm';
+
+import {
+  ActiveWorkspacesCommandOptions,
+  ActiveWorkspacesCommandRunner,
+} from 'src/database/commands/active-workspaces.command';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity';
+
+@Command({
+  name: 'upgrade-0.31:clean-views-with-deleted-object-metadata',
+  description: 'Clean views with deleted object metadata',
+})
+export class CleanViewsWithDeletedObjectMetadataCommand extends ActiveWorkspacesCommandRunner {
+  constructor(
+    @InjectRepository(Workspace, 'core')
+    protected readonly workspaceRepository: Repository<Workspace>,
+    @InjectRepository(ObjectMetadataEntity, 'metadata')
+    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
+    private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+  ) {
+    super(workspaceRepository);
+  }
+
+  async executeActiveWorkspacesCommand(
+    _passedParam: string[],
+    _options: ActiveWorkspacesCommandOptions,
+    workspaceIds: string[],
+  ): Promise<void> {
+    this.logger.log('Running command to fix migration');
+
+    for (const workspaceId of workspaceIds) {
+      this.logger.log(`Running command for workspace ${workspaceId}`);
+
+      try {
+        this.logger.log(chalk.green(`Cleaning views of ${workspaceId}.`));
+
+        await this.cleanViewsWithDeletedObjectMetadata(
+          workspaceId,
+          _options.dryRun ?? false,
+        );
+
+        await this.twentyORMGlobalManager.destroyDataSourceForWorkspace(
+          workspaceId,
+        );
+      } catch (error) {
+        this.logger.log(
+          chalk.red(
+            `Running command on workspace ${workspaceId} failed with error: ${error}`,
+          ),
+        );
+        continue;
+      } finally {
+        this.logger.log(
+          chalk.green(`Finished running command for workspace ${workspaceId}.`),
+        );
+      }
+
+      this.logger.log(chalk.green(`Command completed!`));
+    }
+  }
+
+  private async cleanViewsWithDeletedObjectMetadata(
+    workspaceId: string,
+    dryRun: boolean,
+  ): Promise<void> {
+    const viewRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace<ViewWorkspaceEntity>(
+        workspaceId,
+        'view',
+        false,
+      );
+
+    const allViews = await viewRepository.find();
+
+    const viewObjectMetadataIds = allViews.map((view) => view.objectMetadataId);
+
+    const objectMetadataEntities = await this.objectMetadataRepository.find({
+      where: {
+        id: In(viewObjectMetadataIds),
+      },
+    });
+
+    const validObjectMetadataIds = new Set(
+      objectMetadataEntities.map((entity) => entity.id),
+    );
+
+    const viewIdsToDelete = allViews
+      .filter((view) => !validObjectMetadataIds.has(view.objectMetadataId))
+      .map((view) => view.id);
+
+    if (dryRun) {
+      this.logger.log(
+        chalk.green(
+          `Found ${viewIdsToDelete.length} views to clean in workspace ${workspaceId}.`,
+        ),
+      );
+    }
+
+    if (viewIdsToDelete.length > 0 && !dryRun) {
+      await viewRepository.delete(viewIdsToDelete);
+      this.logger.log(chalk.green(`Cleaning ${viewIdsToDelete.length} views.`));
+    }
+
+    if (viewIdsToDelete.length === 0) {
+      this.logger.log(chalk.green(`No views to clean.`));
+    }
+  }
+}

packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.command.ts

@@ -5,7 +5,9 @@ import { Repository } from 'typeorm';
 
 import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command';
 import { BackfillWorkspaceFavoritesCommand } from 'src/database/commands/upgrade-version/0-31/0-31-backfill-workspace-favorites.command';
+import { CleanViewsWithDeletedObjectMetadataCommand } from 'src/database/commands/upgrade-version/0-31/0-31-clean-views-with-deleted-object-metadata.command';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command';
 
 interface UpdateTo0_31CommandOptions {
   workspaceId?: string;
@@ -19,7 +21,9 @@ export class UpgradeTo0_31Command extends ActiveWorkspacesCommandRunner {
   constructor(
     @InjectRepository(Workspace, 'core')
     protected readonly workspaceRepository: Repository<Workspace>,
+    private readonly syncWorkspaceMetadataCommand: SyncWorkspaceMetadataCommand,
     private readonly backfillWorkspaceFavoritesCommand: BackfillWorkspaceFavoritesCommand,
+    private readonly cleanViewsWithDeletedObjectMetadataCommand: CleanViewsWithDeletedObjectMetadataCommand,
   ) {
     super(workspaceRepository);
   }
@@ -29,12 +33,23 @@ export class UpgradeTo0_31Command extends ActiveWorkspacesCommandRunner {
     options: UpdateTo0_31CommandOptions,
     workspaceIds: string[],
   ): Promise<void> {
-    await this.backfillWorkspaceFavoritesCommand.executeActiveWorkspacesCommand(
+    await this.syncWorkspaceMetadataCommand.executeActiveWorkspacesCommand(
       passedParam,
       {
         ...options,
+        force: true,
       },
       workspaceIds,
     );
+    await this.cleanViewsWithDeletedObjectMetadataCommand.executeActiveWorkspacesCommand(
+      passedParam,
+      options,
+      workspaceIds,
+    );
+    await this.backfillWorkspaceFavoritesCommand.executeActiveWorkspacesCommand(
+      passedParam,
+      options,
+      workspaceIds,
+    );
   }
 }

packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.module.ts

@@ -2,11 +2,22 @@ import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 
 import { BackfillWorkspaceFavoritesCommand } from 'src/database/commands/upgrade-version/0-31/0-31-backfill-workspace-favorites.command';
+import { CleanViewsWithDeletedObjectMetadataCommand } from 'src/database/commands/upgrade-version/0-31/0-31-clean-views-with-deleted-object-metadata.command';
 import { UpgradeTo0_31Command } from 'src/database/commands/upgrade-version/0-31/0-31-upgrade-version.command';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module';
 
 @Module({
-  imports: [TypeOrmModule.forFeature([Workspace], 'core')],
+  imports: [
-  providers: [UpgradeTo0_31Command, BackfillWorkspaceFavoritesCommand],
+    TypeOrmModule.forFeature([Workspace], 'core'),
+    TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'),
+    WorkspaceSyncMetadataCommandsModule,
+  ],
+  providers: [
+    UpgradeTo0_31Command,
+    BackfillWorkspaceFavoritesCommand,
+    CleanViewsWithDeletedObjectMetadataCommand,
+  ],
 })
 export class UpgradeTo0_31CommandModule {}

packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts

@@ -10,7 +10,6 @@ import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm';
 
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
-import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
 import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
 import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
@@ -44,7 +43,6 @@ import { UpdateObjectPayload } from './dtos/update-object.input';
         WorkspaceMigrationModule,
         WorkspaceMigrationRunnerModule,
         WorkspaceMetadataVersionModule,
-        FeatureFlagModule,
         RemoteTableRelationsModule,
       ],
       services: [ObjectMetadataService],

packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts

@@ -10,7 +10,6 @@ import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm';
 import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface';
 
 import { TypeORMService } from 'src/database/typeorm/typeorm.service';
-import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import {
   FieldMetadataEntity,
@@ -60,6 +59,7 @@ import {
   createRelationDeterministicUuid,
 } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util';
 import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity';
+import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity';
 
 import { ObjectMetadataEntity } from './object-metadata.entity';
 
@@ -85,7 +85,6 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
     private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
     private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
     private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
-    private readonly featureFlagService: FeatureFlagService,
   ) {
     super(objectMetadataRepository);
   }
@@ -143,6 +142,24 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
       await this.deleteAllRelationsAndDropTable(objectMetadata, workspaceId);
     }
 
+    // DELETE VIEWS
+    const viewRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace<ViewWorkspaceEntity>(
+        workspaceId,
+        'view',
+      );
+
+    const views = await viewRepository.find({
+      where: {
+        objectMetadataId: objectMetadata.id,
+      },
+    });
+
+    if (views.length > 0) {
+      await viewRepository.delete(views.map((view) => view.id));
+    }
+
+    // DELETE OBJECT
     await this.objectMetadataRepository.delete(objectMetadata.id);
 
     await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(