From 8964d26d5bc3f07e68dff63a01948a58a006ed13 Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Thu, 19 Sep 2024 18:13:07 +0200 Subject: [PATCH] Clean views without object metadata (#7153) Add command for cleaning + clean on object deletion --- ...31-backfill-workspace-favorites.command.ts | 35 +++++- ...ws-with-deleted-object-metadata.command.ts | 115 ++++++++++++++++++ .../0-31/0-31-upgrade-version.command.ts | 17 ++- .../0-31/0-31-upgrade-version.module.ts | 15 ++- .../object-metadata/object-metadata.module.ts | 2 - .../object-metadata.service.ts | 21 +++- 6 files changed, 195 insertions(+), 10 deletions(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-clean-views-with-deleted-object-metadata.command.ts diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-backfill-workspace-favorites.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-backfill-workspace-favorites.command.ts index c651e27de..ed94537b0 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-backfill-workspace-favorites.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-backfill-workspace-favorites.command.ts @@ -2,13 +2,14 @@ import { InjectRepository } from '@nestjs/typeorm'; import chalk from 'chalk'; import { Command } from 'nest-commander'; -import { Repository } from 'typeorm'; +import { In, Repository } from 'typeorm'; import { ActiveWorkspacesCommandOptions, ActiveWorkspacesCommandRunner, } from 'src/database/commands/active-workspaces.command'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity'; @@ -21,6 +22,8 @@ export class BackfillWorkspaceFavoritesCommand extends ActiveWorkspacesCommandRu constructor( @InjectRepository(Workspace, 'core') protected readonly workspaceRepository: Repository, + @InjectRepository(ObjectMetadataEntity, 'metadata') + private readonly objectMetadataRepository: Repository, private readonly twentyORMGlobalManager: TwentyORMGlobalManager, ) { super(workspaceRepository); @@ -37,11 +40,17 @@ export class BackfillWorkspaceFavoritesCommand extends ActiveWorkspacesCommandRu this.logger.log(`Running command for workspace ${workspaceId}`); try { - const workspaceIndexViews = await this.getIndexViews(workspaceId); + const allWorkspaceIndexViews = await this.getIndexViews(workspaceId); + + const activeWorkspaceIndexViews = + await this.filterViewsWithoutObjectMetadata( + workspaceId, + allWorkspaceIndexViews, + ); await this.createViewWorkspaceFavorites( workspaceId, - workspaceIndexViews.map((view) => view.id), + activeWorkspaceIndexViews.map((view) => view.id), ); this.logger.log( @@ -85,6 +94,26 @@ export class BackfillWorkspaceFavoritesCommand extends ActiveWorkspacesCommandRu }); } + private async filterViewsWithoutObjectMetadata( + workspaceId: string, + views: ViewWorkspaceEntity[], + ): Promise { + const viewObjectMetadataIds = views.map((view) => view.objectMetadataId); + + const objectMetadataEntities = await this.objectMetadataRepository.find({ + where: { + workspaceId, + id: In(viewObjectMetadataIds), + }, + }); + + const objectMetadataIds = new Set( + objectMetadataEntities.map((entity) => entity.id), + ); + + return views.filter((view) => objectMetadataIds.has(view.objectMetadataId)); + } + private async createViewWorkspaceFavorites( workspaceId: string, viewIds: string[], diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-clean-views-with-deleted-object-metadata.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-clean-views-with-deleted-object-metadata.command.ts new file mode 100644 index 000000000..727d2a864 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-clean-views-with-deleted-object-metadata.command.ts @@ -0,0 +1,115 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import chalk from 'chalk'; +import { Command } from 'nest-commander'; +import { In, Repository } from 'typeorm'; + +import { + ActiveWorkspacesCommandOptions, + ActiveWorkspacesCommandRunner, +} from 'src/database/commands/active-workspaces.command'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity'; + +@Command({ + name: 'upgrade-0.31:clean-views-with-deleted-object-metadata', + description: 'Clean views with deleted object metadata', +}) +export class CleanViewsWithDeletedObjectMetadataCommand extends ActiveWorkspacesCommandRunner { + constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + @InjectRepository(ObjectMetadataEntity, 'metadata') + private readonly objectMetadataRepository: Repository, + private readonly twentyORMGlobalManager: TwentyORMGlobalManager, + ) { + super(workspaceRepository); + } + + async executeActiveWorkspacesCommand( + _passedParam: string[], + _options: ActiveWorkspacesCommandOptions, + workspaceIds: string[], + ): Promise { + this.logger.log('Running command to fix migration'); + + for (const workspaceId of workspaceIds) { + this.logger.log(`Running command for workspace ${workspaceId}`); + + try { + this.logger.log(chalk.green(`Cleaning views of ${workspaceId}.`)); + + await this.cleanViewsWithDeletedObjectMetadata( + workspaceId, + _options.dryRun ?? false, + ); + + await this.twentyORMGlobalManager.destroyDataSourceForWorkspace( + workspaceId, + ); + } catch (error) { + this.logger.log( + chalk.red( + `Running command on workspace ${workspaceId} failed with error: ${error}`, + ), + ); + continue; + } finally { + this.logger.log( + chalk.green(`Finished running command for workspace ${workspaceId}.`), + ); + } + + this.logger.log(chalk.green(`Command completed!`)); + } + } + + private async cleanViewsWithDeletedObjectMetadata( + workspaceId: string, + dryRun: boolean, + ): Promise { + const viewRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'view', + false, + ); + + const allViews = await viewRepository.find(); + + const viewObjectMetadataIds = allViews.map((view) => view.objectMetadataId); + + const objectMetadataEntities = await this.objectMetadataRepository.find({ + where: { + id: In(viewObjectMetadataIds), + }, + }); + + const validObjectMetadataIds = new Set( + objectMetadataEntities.map((entity) => entity.id), + ); + + const viewIdsToDelete = allViews + .filter((view) => !validObjectMetadataIds.has(view.objectMetadataId)) + .map((view) => view.id); + + if (dryRun) { + this.logger.log( + chalk.green( + `Found ${viewIdsToDelete.length} views to clean in workspace ${workspaceId}.`, + ), + ); + } + + if (viewIdsToDelete.length > 0 && !dryRun) { + await viewRepository.delete(viewIdsToDelete); + this.logger.log(chalk.green(`Cleaning ${viewIdsToDelete.length} views.`)); + } + + if (viewIdsToDelete.length === 0) { + this.logger.log(chalk.green(`No views to clean.`)); + } + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.command.ts index df03c303c..e53dd5e4b 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.command.ts @@ -5,7 +5,9 @@ import { Repository } from 'typeorm'; import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command'; import { BackfillWorkspaceFavoritesCommand } from 'src/database/commands/upgrade-version/0-31/0-31-backfill-workspace-favorites.command'; +import { CleanViewsWithDeletedObjectMetadataCommand } from 'src/database/commands/upgrade-version/0-31/0-31-clean-views-with-deleted-object-metadata.command'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command'; interface UpdateTo0_31CommandOptions { workspaceId?: string; @@ -19,7 +21,9 @@ export class UpgradeTo0_31Command extends ActiveWorkspacesCommandRunner { constructor( @InjectRepository(Workspace, 'core') protected readonly workspaceRepository: Repository, + private readonly syncWorkspaceMetadataCommand: SyncWorkspaceMetadataCommand, private readonly backfillWorkspaceFavoritesCommand: BackfillWorkspaceFavoritesCommand, + private readonly cleanViewsWithDeletedObjectMetadataCommand: CleanViewsWithDeletedObjectMetadataCommand, ) { super(workspaceRepository); } @@ -29,12 +33,23 @@ export class UpgradeTo0_31Command extends ActiveWorkspacesCommandRunner { options: UpdateTo0_31CommandOptions, workspaceIds: string[], ): Promise { - await this.backfillWorkspaceFavoritesCommand.executeActiveWorkspacesCommand( + await this.syncWorkspaceMetadataCommand.executeActiveWorkspacesCommand( passedParam, { ...options, + force: true, }, workspaceIds, ); + await this.cleanViewsWithDeletedObjectMetadataCommand.executeActiveWorkspacesCommand( + passedParam, + options, + workspaceIds, + ); + await this.backfillWorkspaceFavoritesCommand.executeActiveWorkspacesCommand( + passedParam, + options, + workspaceIds, + ); } } diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.module.ts index 93e3f965c..32623fe43 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-31-upgrade-version.module.ts @@ -2,11 +2,22 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { BackfillWorkspaceFavoritesCommand } from 'src/database/commands/upgrade-version/0-31/0-31-backfill-workspace-favorites.command'; +import { CleanViewsWithDeletedObjectMetadataCommand } from 'src/database/commands/upgrade-version/0-31/0-31-clean-views-with-deleted-object-metadata.command'; import { UpgradeTo0_31Command } from 'src/database/commands/upgrade-version/0-31/0-31-upgrade-version.command'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module'; @Module({ - imports: [TypeOrmModule.forFeature([Workspace], 'core')], - providers: [UpgradeTo0_31Command, BackfillWorkspaceFavoritesCommand], + imports: [ + TypeOrmModule.forFeature([Workspace], 'core'), + TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'), + WorkspaceSyncMetadataCommandsModule, + ], + providers: [ + UpgradeTo0_31Command, + BackfillWorkspaceFavoritesCommand, + CleanViewsWithDeletedObjectMetadataCommand, + ], }) export class UpgradeTo0_31CommandModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts index 2f750d965..94a721dec 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts @@ -10,7 +10,6 @@ import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; -import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; @@ -44,7 +43,6 @@ import { UpdateObjectPayload } from './dtos/update-object.input'; WorkspaceMigrationModule, WorkspaceMigrationRunnerModule, WorkspaceMetadataVersionModule, - FeatureFlagModule, RemoteTableRelationsModule, ], services: [ObjectMetadataService], diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index d6146870e..c0240c6c2 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -10,7 +10,6 @@ import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm'; import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; -import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { FieldMetadataEntity, @@ -60,6 +59,7 @@ import { createRelationDeterministicUuid, } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; +import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity'; import { ObjectMetadataEntity } from './object-metadata.entity'; @@ -85,7 +85,6 @@ export class ObjectMetadataService extends TypeOrmQueryService( + workspaceId, + 'view', + ); + + const views = await viewRepository.find({ + where: { + objectMetadataId: objectMetadata.id, + }, + }); + + if (views.length > 0) { + await viewRepository.delete(views.map((view) => view.id)); + } + + // DELETE OBJECT await this.objectMetadataRepository.delete(objectMetadata.id); await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(