mirror of
				https://github.com/lingble/twenty.git
				synced 2025-10-31 04:37:56 +00:00 
			
		
		
		
	Refactor metadata caching (#7011)
This PR introduces the following changes: - add the metadataVersion to all our metadata cache keys to ease troubleshooting: <img width="1146" alt="image" src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12"> - introduce a cache recompute lock to avoid overloading the database to recompute the cache many time
This commit is contained in:
		 Charles Bochet
					Charles Bochet
				
			
				
					committed by
					
						 Charles Bochet
						Charles Bochet
					
				
			
			
				
	
			
			
			 Charles Bochet
						Charles Bochet
					
				
			
						parent
						
							9b46e8c663
						
					
				
				
					commit
					3c4168759a
				
			| @@ -20,9 +20,9 @@ import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphq | ||||
| import { RestApiModule } from 'src/engine/api/rest/rest-api.module'; | ||||
| import { MessageQueueDriverType } from 'src/engine/core-modules/message-queue/interfaces'; | ||||
| import { MessageQueueModule } from 'src/engine/core-modules/message-queue/message-queue.module'; | ||||
| import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; | ||||
| import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware'; | ||||
| import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; | ||||
| import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; | ||||
| import { ModulesModule } from 'src/modules/modules.module'; | ||||
|  | ||||
| import { CoreEngineModule } from './engine/core-modules/core-engine.module'; | ||||
| @@ -51,7 +51,7 @@ import { CoreEngineModule } from './engine/core-modules/core-engine.module'; | ||||
|     // Modules module, contains all business logic modules | ||||
|     ModulesModule, | ||||
|     // Needed for the user workspace middleware | ||||
|     WorkspaceMetadataVersionModule, | ||||
|     WorkspaceCacheStorageModule, | ||||
|     // Api modules | ||||
|     CoreGraphQLApiModule, | ||||
|     MetadataGraphQLApiModule, | ||||
|   | ||||
| @@ -1,11 +1,17 @@ | ||||
| import { Module } from '@nestjs/common'; | ||||
| import { TypeOrmModule } from '@nestjs/typeorm'; | ||||
|  | ||||
| import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module'; | ||||
| import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; | ||||
| import { DataSeedDemoWorkspaceService } from 'src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service'; | ||||
| import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module'; | ||||
| import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; | ||||
| import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; | ||||
|  | ||||
| @Module({ | ||||
|   imports: [WorkspaceManagerModule, EnvironmentModule], | ||||
|   imports: [ | ||||
|     WorkspaceManagerModule, | ||||
|     EnvironmentModule, | ||||
|     TypeOrmModule.forFeature([Workspace], 'core'), | ||||
|   ], | ||||
|   providers: [DataSeedDemoWorkspaceService], | ||||
|   exports: [DataSeedDemoWorkspaceService], | ||||
| }) | ||||
|   | ||||
| @@ -1,18 +1,24 @@ | ||||
| import { Injectable } from '@nestjs/common'; | ||||
| import { InjectRepository } from '@nestjs/typeorm'; | ||||
|  | ||||
| import { Repository } from 'typeorm'; | ||||
|  | ||||
| import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; | ||||
| import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; | ||||
| import { | ||||
|   deleteCoreSchema, | ||||
|   seedCoreSchema, | ||||
| } from 'src/database/typeorm-seeds/core/demo'; | ||||
| import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource'; | ||||
| import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; | ||||
| import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; | ||||
| import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; | ||||
|  | ||||
| @Injectable() | ||||
| export class DataSeedDemoWorkspaceService { | ||||
|   constructor( | ||||
|     private readonly environmentService: EnvironmentService, | ||||
|     private readonly workspaceManagerService: WorkspaceManagerService, | ||||
|     @InjectRepository(Workspace, 'core') | ||||
|     protected readonly workspaceRepository: Repository<Workspace>, | ||||
|   ) {} | ||||
|  | ||||
|   async seedDemo(): Promise<void> { | ||||
| @@ -27,8 +33,14 @@ export class DataSeedDemoWorkspaceService { | ||||
|         ); | ||||
|       } | ||||
|       for (const workspaceId of demoWorkspaceIds) { | ||||
|         await deleteCoreSchema(rawDataSource, workspaceId); | ||||
|         const existingWorkspaces = await this.workspaceRepository.findBy({ | ||||
|           id: workspaceId, | ||||
|         }); | ||||
|  | ||||
|         if (existingWorkspaces.length > 0) { | ||||
|           await this.workspaceManagerService.delete(workspaceId); | ||||
|           await deleteCoreSchema(rawDataSource, workspaceId); | ||||
|         } | ||||
|  | ||||
|         await seedCoreSchema(rawDataSource, workspaceId); | ||||
|         await this.workspaceManagerService.initDemo(workspaceId); | ||||
|   | ||||
| @@ -9,7 +9,7 @@ import { | ||||
|   Workspace, | ||||
|   WorkspaceActivationStatus, | ||||
| } from 'src/engine/core-modules/workspace/workspace.entity'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; | ||||
| import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; | ||||
| import { MessageDirection } from 'src/modules/messaging/common/enums/message-direction.enum'; | ||||
| import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity'; | ||||
|   | ||||
| @@ -6,7 +6,7 @@ import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace- | ||||
| import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory'; | ||||
| import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; | ||||
| import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; | ||||
| import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; | ||||
|  | ||||
| describe('WorkspaceSchemaFactory', () => { | ||||
| @@ -41,7 +41,7 @@ describe('WorkspaceSchemaFactory', () => { | ||||
|           useValue: {}, | ||||
|         }, | ||||
|         { | ||||
|           provide: WorkspaceMetadataVersionService, | ||||
|           provide: WorkspaceMetadataCacheService, | ||||
|           useValue: {}, | ||||
|         }, | ||||
|       ], | ||||
|   | ||||
| @@ -4,25 +4,18 @@ import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars- | ||||
| import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver-builder.module'; | ||||
| import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module'; | ||||
| import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module'; | ||||
| import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; | ||||
| import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; | ||||
| import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; | ||||
|  | ||||
| import { WorkspaceSchemaFactory } from './workspace-schema.factory'; | ||||
|  | ||||
| @Module({ | ||||
|   imports: [ | ||||
|     // TODO: Seems like it's breaking /metadata query and mutation arguments | ||||
|     // we should investigate this issue | ||||
|     // GraphQLModule.forRootAsync<YogaDriverConfig>({ | ||||
|     //   driver: YogaDriver, | ||||
|     //   imports: [CoreEngineModule, GraphQLConfigModule], | ||||
|     //   useClass: GraphQLConfigService, | ||||
|     // }), | ||||
|     MetadataEngineModule, | ||||
|     WorkspaceSchemaBuilderModule, | ||||
|     WorkspaceResolverBuilderModule, | ||||
|     WorkspaceCacheStorageModule, | ||||
|     WorkspaceMetadataVersionModule, | ||||
|     WorkspaceMetadataCacheModule, | ||||
|   ], | ||||
|   providers: [WorkspaceSchemaFactory, ScalarsExplorerService], | ||||
|   exports: [WorkspaceSchemaFactory], | ||||
|   | ||||
| @@ -19,4 +19,5 @@ export enum GraphqlQueryRunnerExceptionCode { | ||||
|   RECORD_NOT_FOUND = 'RECORD_NOT_FOUND', | ||||
|   INVALID_ARGS_FIRST = 'INVALID_ARGS_FIRST', | ||||
|   INVALID_ARGS_LAST = 'INVALID_ARGS_LAST', | ||||
|   METADATA_CACHE_VERSION_NOT_FOUND = 'METADATA_CACHE_VERSION_NOT_FOUND', | ||||
| } | ||||
|   | ||||
| @@ -4,25 +4,27 @@ import { makeExecutableSchema } from '@graphql-tools/schema'; | ||||
| import { GraphQLSchema, printSchema } from 'graphql'; | ||||
| import { gql } from 'graphql-tag'; | ||||
|  | ||||
| import { | ||||
|   GraphqlQueryRunnerException, | ||||
|   GraphqlQueryRunnerExceptionCode, | ||||
| } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception'; | ||||
| import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service'; | ||||
| import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories'; | ||||
| import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory'; | ||||
| import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory'; | ||||
| import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; | ||||
| import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; | ||||
| import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; | ||||
| import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; | ||||
| @Injectable() | ||||
| export class WorkspaceSchemaFactory { | ||||
|   constructor( | ||||
|     private readonly dataSourceService: DataSourceService, | ||||
|     private readonly objectMetadataService: ObjectMetadataService, | ||||
|     private readonly scalarsExplorerService: ScalarsExplorerService, | ||||
|     private readonly workspaceGraphQLSchemaFactory: WorkspaceGraphQLSchemaFactory, | ||||
|     private readonly workspaceResolverFactory: WorkspaceResolverFactory, | ||||
|     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, | ||||
|     private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, | ||||
|     private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, | ||||
|   ) {} | ||||
|  | ||||
|   async createGraphQLSchema(authContext: AuthContext): Promise<GraphQLSchema> { | ||||
| @@ -35,42 +37,50 @@ export class WorkspaceSchemaFactory { | ||||
|         authContext.workspace.id, | ||||
|       ); | ||||
|  | ||||
|     // Can'f find any data sources for this workspace | ||||
|     if (!dataSourcesMetadata || dataSourcesMetadata.length === 0) { | ||||
|       return new GraphQLSchema({}); | ||||
|     } | ||||
|  | ||||
|     // Validate cache version | ||||
|     await this.workspaceMetadataVersionService.flushCacheIfMetadataVersionIsOutdated( | ||||
|     const currentCacheVersion = | ||||
|       await this.workspaceCacheStorageService.getMetadataVersion( | ||||
|         authContext.workspace.id, | ||||
|       ); | ||||
|  | ||||
|     // Get object metadata from cache | ||||
|     let objectMetadataCollection = | ||||
|     if (currentCacheVersion === undefined) { | ||||
|       await this.workspaceMetadataCacheService.recomputeMetadataCache( | ||||
|         authContext.workspace.id, | ||||
|       ); | ||||
|       throw new GraphqlQueryRunnerException( | ||||
|         'Metadata cache version not found', | ||||
|         GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND, | ||||
|       ); | ||||
|     } | ||||
|  | ||||
|     const objectMetadataCollection = | ||||
|       await this.workspaceCacheStorageService.getObjectMetadataCollection( | ||||
|         authContext.workspace.id, | ||||
|         currentCacheVersion, | ||||
|       ); | ||||
|  | ||||
|     // If object metadata is not cached, get it from the database | ||||
|     if (!objectMetadataCollection) { | ||||
|       objectMetadataCollection = | ||||
|         await this.objectMetadataService.findManyWithinWorkspace( | ||||
|       await this.workspaceMetadataCacheService.recomputeMetadataCache( | ||||
|         authContext.workspace.id, | ||||
|       ); | ||||
|  | ||||
|       await this.workspaceCacheStorageService.setObjectMetadataCollection( | ||||
|         authContext.workspace.id, | ||||
|         objectMetadataCollection, | ||||
|       throw new GraphqlQueryRunnerException( | ||||
|         'Object metadata collection not found', | ||||
|         GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND, | ||||
|       ); | ||||
|     } | ||||
|  | ||||
|     // Get typeDefs from cache | ||||
|     let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs( | ||||
|       authContext.workspace.id, | ||||
|       currentCacheVersion, | ||||
|     ); | ||||
|     let usedScalarNames = | ||||
|       await this.workspaceCacheStorageService.getGraphQLUsedScalarNames( | ||||
|         authContext.workspace.id, | ||||
|         currentCacheVersion, | ||||
|       ); | ||||
|  | ||||
|     // If typeDefs are not cached, generate them | ||||
| @@ -87,10 +97,12 @@ export class WorkspaceSchemaFactory { | ||||
|  | ||||
|       await this.workspaceCacheStorageService.setGraphQLTypeDefs( | ||||
|         authContext.workspace.id, | ||||
|         currentCacheVersion, | ||||
|         typeDefs, | ||||
|       ); | ||||
|       await this.workspaceCacheStorageService.setGraphQLUsedScalarNames( | ||||
|         authContext.workspace.id, | ||||
|         currentCacheVersion, | ||||
|         usedScalarNames, | ||||
|       ); | ||||
|     } | ||||
|   | ||||
| @@ -16,7 +16,7 @@ import { User } from 'src/engine/core-modules/user/user.entity'; | ||||
| import { WorkspaceWorkspaceMemberListener } from 'src/engine/core-modules/workspace/workspace-workspace-member.listener'; | ||||
| import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver'; | ||||
| import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; | ||||
| import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; | ||||
| import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; | ||||
| import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; | ||||
|  | ||||
| import { workspaceAutoResolverOpts } from './workspace.auto-resolver-opts'; | ||||
| @@ -32,7 +32,7 @@ import { WorkspaceService } from './services/workspace.service'; | ||||
|         BillingModule, | ||||
|         FileModule, | ||||
|         FileUploadModule, | ||||
|         WorkspaceMetadataVersionModule, | ||||
|         WorkspaceMetadataCacheModule, | ||||
|         NestjsQueryTypeOrmModule.forFeature( | ||||
|           [User, Workspace, UserWorkspace, FeatureFlagEntity], | ||||
|           'core', | ||||
|   | ||||
| @@ -26,7 +26,6 @@ import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator'; | ||||
| import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator'; | ||||
| import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; | ||||
| import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { assert } from 'src/utils/assert'; | ||||
| import { streamToBuffer } from 'src/utils/stream-to-buffer'; | ||||
|  | ||||
| @@ -39,7 +38,6 @@ import { WorkspaceService } from './services/workspace.service'; | ||||
| export class WorkspaceResolver { | ||||
|   constructor( | ||||
|     private readonly workspaceService: WorkspaceService, | ||||
|     private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, | ||||
|     private readonly userWorkspaceService: UserWorkspaceService, | ||||
|     private readonly fileUploadService: FileUploadService, | ||||
|     private readonly fileService: FileService, | ||||
|   | ||||
| @@ -41,7 +41,7 @@ import { NameTooLongException } from 'src/engine/metadata-modules/utils/exceptio | ||||
| import { exceedsDatabaseIdentifierMaximumLength } from 'src/engine/metadata-modules/utils/validate-database-identifier-length.utils'; | ||||
| import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils'; | ||||
| import { validateMetadataNameValidityOrThrow as validateFieldNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; | ||||
| import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; | ||||
| import { | ||||
|   WorkspaceMigrationColumnActionType, | ||||
|   | ||||
| @@ -34,7 +34,7 @@ import { | ||||
| import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/types/relation-to-delete'; | ||||
| import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service'; | ||||
| import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; | ||||
| import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; | ||||
| import { | ||||
|   WorkspaceMigrationColumnActionType, | ||||
|   | ||||
| @@ -21,7 +21,7 @@ import { | ||||
| import { InvalidStringException } from 'src/engine/metadata-modules/utils/exceptions/invalid-string.exception'; | ||||
| import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils'; | ||||
| import { validateMetadataNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; | ||||
| import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; | ||||
| import { | ||||
|   WorkspaceMigrationColumnActionType, | ||||
|   | ||||
| @@ -11,7 +11,7 @@ import { | ||||
| } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.exception'; | ||||
| import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; | ||||
| import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; | ||||
| import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; | ||||
| import { | ||||
|   ReferencedTable, | ||||
|   | ||||
| @@ -36,7 +36,7 @@ import { | ||||
|   mapUdtNameToFieldType, | ||||
| } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; | ||||
| import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; | ||||
| import { | ||||
|   WorkspaceMigrationColumnAction, | ||||
|   WorkspaceMigrationColumnActionType, | ||||
|   | ||||
| @@ -0,0 +1,12 @@ | ||||
| import { CustomException } from 'src/utils/custom-exception'; | ||||
|  | ||||
| export class WorkspaceMetadataCacheException extends CustomException { | ||||
|   code: WorkspaceMetadataCacheExceptionCode; | ||||
|   constructor(message: string, code: WorkspaceMetadataCacheExceptionCode) { | ||||
|     super(message, code); | ||||
|   } | ||||
| } | ||||
|  | ||||
| export enum WorkspaceMetadataCacheExceptionCode { | ||||
|   METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND', | ||||
| } | ||||
| @@ -0,0 +1,112 @@ | ||||
| import { Injectable, Logger } from '@nestjs/common'; | ||||
| import { InjectRepository } from '@nestjs/typeorm'; | ||||
|  | ||||
| import { Repository } from 'typeorm'; | ||||
|  | ||||
| import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; | ||||
| import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; | ||||
| import { | ||||
|   WorkspaceMetadataCacheException, | ||||
|   WorkspaceMetadataCacheExceptionCode, | ||||
| } from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception'; | ||||
| import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; | ||||
|  | ||||
| @Injectable() | ||||
| export class WorkspaceMetadataCacheService { | ||||
|   logger = new Logger(WorkspaceMetadataCacheService.name); | ||||
|  | ||||
|   constructor( | ||||
|     @InjectRepository(Workspace, 'core') | ||||
|     private readonly workspaceRepository: Repository<Workspace>, | ||||
|     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, | ||||
|     @InjectRepository(ObjectMetadataEntity, 'metadata') | ||||
|     private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>, | ||||
|   ) {} | ||||
|  | ||||
|   async recomputeMetadataCache( | ||||
|     workspaceId: string, | ||||
|     force = false, | ||||
|   ): Promise<void> { | ||||
|     const currentCacheVersion = | ||||
|       await this.getMetadataVersionFromCache(workspaceId); | ||||
|  | ||||
|     const currentDatabaseVersion = | ||||
|       await this.getMetadataVersionFromDatabase(workspaceId); | ||||
|  | ||||
|     if (currentDatabaseVersion === undefined) { | ||||
|       throw new WorkspaceMetadataCacheException( | ||||
|         'Metadata version not found in the database', | ||||
|         WorkspaceMetadataCacheExceptionCode.METADATA_VERSION_NOT_FOUND, | ||||
|       ); | ||||
|     } | ||||
|  | ||||
|     if (!force && currentCacheVersion === currentDatabaseVersion) { | ||||
|       return; | ||||
|     } | ||||
|  | ||||
|     const isAlreadyCaching = | ||||
|       await this.workspaceCacheStorageService.getObjectMetadataCollectionOngoingCachingLock( | ||||
|         workspaceId, | ||||
|         currentDatabaseVersion, | ||||
|       ); | ||||
|  | ||||
|     if (isAlreadyCaching) { | ||||
|       return; | ||||
|     } | ||||
|  | ||||
|     if (currentCacheVersion !== undefined) { | ||||
|       this.workspaceCacheStorageService.flush(workspaceId, currentCacheVersion); | ||||
|     } | ||||
|  | ||||
|     await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock( | ||||
|       workspaceId, | ||||
|       currentDatabaseVersion, | ||||
|     ); | ||||
|  | ||||
|     await this.workspaceCacheStorageService.setMetadataVersion( | ||||
|       workspaceId, | ||||
|       currentDatabaseVersion, | ||||
|     ); | ||||
|  | ||||
|     const freshObjectMetadataCollection = | ||||
|       await this.objectMetadataRepository.find({ | ||||
|         where: { workspaceId }, | ||||
|         relations: [ | ||||
|           'fields.object', | ||||
|           'fields', | ||||
|           'fields.fromRelationMetadata', | ||||
|           'fields.toRelationMetadata', | ||||
|           'fields.fromRelationMetadata.toObjectMetadata', | ||||
|         ], | ||||
|       }); | ||||
|  | ||||
|     await this.workspaceCacheStorageService.setObjectMetadataCollection( | ||||
|       workspaceId, | ||||
|       currentDatabaseVersion, | ||||
|       freshObjectMetadataCollection, | ||||
|     ); | ||||
|  | ||||
|     await this.workspaceCacheStorageService.removeObjectMetadataCollectionOngoingCachingLock( | ||||
|       workspaceId, | ||||
|       currentDatabaseVersion, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   private async getMetadataVersionFromDatabase( | ||||
|     workspaceId: string, | ||||
|   ): Promise<number | undefined> { | ||||
|     const workspace = await this.workspaceRepository.findOne({ | ||||
|       where: { id: workspaceId }, | ||||
|     }); | ||||
|  | ||||
|     return workspace?.metadataVersion; | ||||
|   } | ||||
|  | ||||
|   private async getMetadataVersionFromCache( | ||||
|     workspaceId: string, | ||||
|   ): Promise<number | undefined> { | ||||
|     return await this.workspaceCacheStorageService.getMetadataVersion( | ||||
|       workspaceId, | ||||
|     ); | ||||
|   } | ||||
| } | ||||
| @@ -0,0 +1,18 @@ | ||||
| import { Module } from '@nestjs/common'; | ||||
| import { TypeOrmModule } from '@nestjs/typeorm'; | ||||
|  | ||||
| import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; | ||||
| import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; | ||||
| import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; | ||||
| import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; | ||||
|  | ||||
| @Module({ | ||||
|   imports: [ | ||||
|     TypeOrmModule.forFeature([Workspace], 'core'), | ||||
|     TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'), | ||||
|     WorkspaceCacheStorageModule, | ||||
|   ], | ||||
|   exports: [WorkspaceMetadataCacheService], | ||||
|   providers: [WorkspaceMetadataCacheService], | ||||
| }) | ||||
| export class WorkspaceMetadataCacheModule {} | ||||
| @@ -0,0 +1,12 @@ | ||||
| import { CustomException } from 'src/utils/custom-exception'; | ||||
|  | ||||
| export class WorkspaceMetadataVersionException extends CustomException { | ||||
|   code: WorkspaceMetadataVersionExceptionCode; | ||||
|   constructor(message: string, code: WorkspaceMetadataVersionExceptionCode) { | ||||
|     super(message, code); | ||||
|   } | ||||
| } | ||||
|  | ||||
| export enum WorkspaceMetadataVersionExceptionCode { | ||||
|   METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND', | ||||
| } | ||||
| @@ -0,0 +1,52 @@ | ||||
| import { Injectable, Logger } from '@nestjs/common'; | ||||
| import { InjectRepository } from '@nestjs/typeorm'; | ||||
|  | ||||
| import { Repository } from 'typeorm'; | ||||
|  | ||||
| import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; | ||||
| import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; | ||||
| import { | ||||
|   WorkspaceMetadataVersionException, | ||||
|   WorkspaceMetadataVersionExceptionCode, | ||||
| } from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception'; | ||||
| import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; | ||||
|  | ||||
| @Injectable() | ||||
| export class WorkspaceMetadataVersionService { | ||||
|   logger = new Logger(WorkspaceMetadataCacheService.name); | ||||
|  | ||||
|   constructor( | ||||
|     @InjectRepository(Workspace, 'core') | ||||
|     private readonly workspaceRepository: Repository<Workspace>, | ||||
|     private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, | ||||
|     private readonly twentyORMGlobalManager: TwentyORMGlobalManager, | ||||
|   ) {} | ||||
|  | ||||
|   async incrementMetadataVersion(workspaceId: string): Promise<void> { | ||||
|     const workspace = await this.workspaceRepository.findOne({ | ||||
|       where: { id: workspaceId }, | ||||
|     }); | ||||
|  | ||||
|     const metadataVersion = workspace?.metadataVersion; | ||||
|  | ||||
|     if (metadataVersion === undefined) { | ||||
|       throw new WorkspaceMetadataVersionException( | ||||
|         'Metadata version not found', | ||||
|         WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND, | ||||
|       ); | ||||
|     } | ||||
|  | ||||
|     const newMetadataVersion = metadataVersion + 1; | ||||
|  | ||||
|     await this.workspaceRepository.update( | ||||
|       { id: workspaceId }, | ||||
|       { metadataVersion: newMetadataVersion }, | ||||
|     ); | ||||
|  | ||||
|     await this.workspaceMetadataCacheService.recomputeMetadataCache( | ||||
|       workspaceId, | ||||
|     ); | ||||
|  | ||||
|     await this.twentyORMGlobalManager.loadDataSourceForWorkspace(workspaceId); | ||||
|   } | ||||
| } | ||||
| @@ -2,13 +2,15 @@ import { Module } from '@nestjs/common'; | ||||
| import { TypeOrmModule } from '@nestjs/typeorm'; | ||||
|  | ||||
| import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; | ||||
| import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; | ||||
|  | ||||
| @Module({ | ||||
|   imports: [ | ||||
|     TypeOrmModule.forFeature([Workspace], 'core'), | ||||
|     WorkspaceCacheStorageModule, | ||||
|     WorkspaceMetadataCacheModule, | ||||
|   ], | ||||
|   exports: [WorkspaceMetadataVersionService], | ||||
|   providers: [WorkspaceMetadataVersionService], | ||||
|   | ||||
| @@ -1,81 +0,0 @@ | ||||
| import { Injectable, Logger } from '@nestjs/common'; | ||||
| import { InjectRepository } from '@nestjs/typeorm'; | ||||
|  | ||||
| import { Repository } from 'typeorm'; | ||||
|  | ||||
| import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; | ||||
| import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; | ||||
|  | ||||
| @Injectable() | ||||
| export class WorkspaceMetadataVersionService { | ||||
|   logger = new Logger(WorkspaceMetadataVersionService.name); | ||||
|  | ||||
|   constructor( | ||||
|     @InjectRepository(Workspace, 'core') | ||||
|     private readonly workspaceRepository: Repository<Workspace>, | ||||
|     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, | ||||
|   ) {} | ||||
|  | ||||
|   async flushCacheIfMetadataVersionIsOutdated( | ||||
|     workspaceId: string, | ||||
|   ): Promise<void> { | ||||
|     const currentVersion = | ||||
|       (await this.workspaceCacheStorageService.getMetadataVersion( | ||||
|         workspaceId, | ||||
|       )) ?? 1; | ||||
|  | ||||
|     let latestVersion = await this.getMetadataVersion(workspaceId); | ||||
|  | ||||
|     if (latestVersion === undefined || currentVersion !== latestVersion) { | ||||
|       this.logger.log( | ||||
|         `Metadata version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`, | ||||
|       ); | ||||
|  | ||||
|       await this.workspaceCacheStorageService.flush(workspaceId); | ||||
|  | ||||
|       latestVersion = await this.incrementMetadataVersion(workspaceId); | ||||
|  | ||||
|       await this.workspaceCacheStorageService.setMetadataVersion( | ||||
|         workspaceId, | ||||
|         latestVersion, | ||||
|       ); | ||||
|     } | ||||
|   } | ||||
|  | ||||
|   async incrementMetadataVersion(workspaceId: string): Promise<number> { | ||||
|     const metadataVersion = (await this.getMetadataVersion(workspaceId)) ?? 0; | ||||
|     const newMetadataVersion = metadataVersion + 1; | ||||
|  | ||||
|     await this.workspaceRepository.update( | ||||
|       { id: workspaceId }, | ||||
|       { metadataVersion: newMetadataVersion }, | ||||
|     ); | ||||
|  | ||||
|     await this.workspaceCacheStorageService.flush(workspaceId); | ||||
|  | ||||
|     await this.workspaceCacheStorageService.setMetadataVersion( | ||||
|       workspaceId, | ||||
|       newMetadataVersion, | ||||
|     ); | ||||
|  | ||||
|     return newMetadataVersion; | ||||
|   } | ||||
|  | ||||
|   async getMetadataVersion(workspaceId: string): Promise<number | undefined> { | ||||
|     const workspace = await this.workspaceRepository.findOne({ | ||||
|       where: { id: workspaceId }, | ||||
|     }); | ||||
|  | ||||
|     return workspace?.metadataVersion; | ||||
|   } | ||||
|  | ||||
|   async resetMetadataVersion(workspaceId: string): Promise<void> { | ||||
|     await this.workspaceRepository.update( | ||||
|       { id: workspaceId }, | ||||
|       { metadataVersion: 1 }, | ||||
|     ); | ||||
|  | ||||
|     await this.workspaceCacheStorageService.flush(workspaceId); | ||||
|     await this.workspaceCacheStorageService.setMetadataVersion(workspaceId, 1); | ||||
|   } | ||||
| } | ||||
| @@ -6,8 +6,8 @@ import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filt | ||||
| import { TokenService } from 'src/engine/core-modules/auth/services/token.service'; | ||||
| import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; | ||||
| import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { handleExceptionAndConvertToGraphQLError } from 'src/engine/utils/global-exception-handler.util'; | ||||
| import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; | ||||
|  | ||||
| class GraphqlTokenValidationProxy { | ||||
|   private tokenService: TokenService; | ||||
| @@ -33,7 +33,7 @@ export class GraphQLHydrateRequestFromTokenMiddleware | ||||
| { | ||||
|   constructor( | ||||
|     private readonly tokenService: TokenService, | ||||
|     private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, | ||||
|     private readonly workspaceStorageCacheService: WorkspaceCacheStorageService, | ||||
|     private readonly exceptionHandlerService: ExceptionHandlerService, | ||||
|   ) {} | ||||
|  | ||||
| @@ -73,7 +73,7 @@ export class GraphQLHydrateRequestFromTokenMiddleware | ||||
|  | ||||
|       data = await graphqlTokenValidationProxy.validateToken(req); | ||||
|       const metadataVersion = | ||||
|         await this.workspaceMetadataVersionService.getMetadataVersion( | ||||
|         await this.workspaceStorageCacheService.getMetadataVersion( | ||||
|           data.workspace.id, | ||||
|         ); | ||||
|  | ||||
|   | ||||
| @@ -0,0 +1,15 @@ | ||||
| import { CustomException } from 'src/utils/custom-exception'; | ||||
|  | ||||
| export class TwentyORMException extends CustomException { | ||||
|   code: TwentyORMExceptionCode; | ||||
|   constructor(message: string, code: TwentyORMExceptionCode) { | ||||
|     super(message, code); | ||||
|   } | ||||
| } | ||||
|  | ||||
| export enum TwentyORMExceptionCode { | ||||
|   METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND', | ||||
|   METADATA_VERSION_MISMATCH = 'METADATA_VERSION_MISMATCH', | ||||
|   METADATA_COLLECTION_NOT_FOUND = 'METADATA_COLLECTION_NOT_FOUND', | ||||
|   WORKSPACE_SCHEMA_NOT_FOUND = 'WORKSPACE_SCHEMA_NOT_FOUND', | ||||
| } | ||||
| @@ -19,6 +19,7 @@ export class EntitySchemaRelationFactory { | ||||
|  | ||||
|   async create( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|     fieldMetadataCollection: FieldMetadataEntity[], | ||||
|   ): Promise<EntitySchemaRelationMap> { | ||||
|     const entitySchemaRelationMap: EntitySchemaRelationMap = {}; | ||||
| @@ -40,6 +41,7 @@ export class EntitySchemaRelationFactory { | ||||
|       const objectMetadataCollection = | ||||
|         await this.workspaceCacheStorageService.getObjectMetadataCollection( | ||||
|           workspaceId, | ||||
|           metadataVersion, | ||||
|         ); | ||||
|  | ||||
|       if (!objectMetadataCollection) { | ||||
|   | ||||
| @@ -17,6 +17,7 @@ export class EntitySchemaFactory { | ||||
|  | ||||
|   async create( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|     objectMetadata: ObjectMetadataEntity, | ||||
|   ): Promise<EntitySchema> { | ||||
|     const columns = this.entitySchemaColumnFactory.create( | ||||
| @@ -26,6 +27,7 @@ export class EntitySchemaFactory { | ||||
|  | ||||
|     const relations = await this.entitySchemaRelationFactory.create( | ||||
|       workspaceId, | ||||
|       metadataVersion, | ||||
|       objectMetadata.fields, | ||||
|     ); | ||||
|  | ||||
|   | ||||
| @@ -11,11 +11,11 @@ export class ScopedWorkspaceContextFactory { | ||||
|  | ||||
|   public create(): { | ||||
|     workspaceId: string | null; | ||||
|     workspaceMetadataVersion: string | null; | ||||
|     workspaceMetadataVersion: number | null; | ||||
|   } { | ||||
|     const workspaceId: string | undefined = | ||||
|       this.request?.['req']?.['workspaceId']; | ||||
|     const workspaceMetadataVersion: string | undefined = | ||||
|     const workspaceMetadataVersion: number | undefined = | ||||
|       this.request?.['req']?.['workspaceMetadataVersion']; | ||||
|  | ||||
|     return { | ||||
|   | ||||
| @@ -1,13 +1,15 @@ | ||||
| import { Injectable } from '@nestjs/common'; | ||||
| import { InjectRepository } from '@nestjs/typeorm'; | ||||
|  | ||||
| import { EntitySchema, Repository } from 'typeorm'; | ||||
| import { EntitySchema } from 'typeorm'; | ||||
|  | ||||
| import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; | ||||
| import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; | ||||
| import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; | ||||
| import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource'; | ||||
| import { | ||||
|   TwentyORMException, | ||||
|   TwentyORMExceptionCode, | ||||
| } from 'src/engine/twenty-orm/exceptions/twenty-orm.exception'; | ||||
| import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; | ||||
| import { CacheManager } from 'src/engine/twenty-orm/storage/cache-manager.storage'; | ||||
| import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; | ||||
| @@ -20,63 +22,56 @@ export class WorkspaceDatasourceFactory { | ||||
|     private readonly dataSourceService: DataSourceService, | ||||
|     private readonly environmentService: EnvironmentService, | ||||
|     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, | ||||
|     private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, | ||||
|     @InjectRepository(ObjectMetadataEntity, 'metadata') | ||||
|     private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>, | ||||
|     private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, | ||||
|     private readonly entitySchemaFactory: EntitySchemaFactory, | ||||
|   ) {} | ||||
|  | ||||
|   public async create( | ||||
|     workspaceId: string, | ||||
|     workspaceMetadataVersion: string | null, | ||||
|     workspaceMetadataVersion: number | null, | ||||
|   ): Promise<WorkspaceDataSource> { | ||||
|     const latestWorkspaceMetadataVersion = | ||||
|       await this.workspaceMetadataVersionService.getMetadataVersion( | ||||
|       await this.workspaceCacheStorageService.getMetadataVersion(workspaceId); | ||||
|  | ||||
|     if (latestWorkspaceMetadataVersion === undefined) { | ||||
|       await this.workspaceMetadataCacheService.recomputeMetadataCache( | ||||
|         workspaceId, | ||||
|       ); | ||||
|       throw new TwentyORMException( | ||||
|         `Metadata version not found for workspace ${workspaceId}`, | ||||
|         TwentyORMExceptionCode.METADATA_VERSION_NOT_FOUND, | ||||
|       ); | ||||
|     } | ||||
|  | ||||
|     const desiredWorkspaceMetadataVersion = | ||||
|       workspaceMetadataVersion ?? latestWorkspaceMetadataVersion; | ||||
|  | ||||
|     if (!desiredWorkspaceMetadataVersion) { | ||||
|       throw new Error( | ||||
|         `Desired workspace metadata version not found while creating workspace data source for workspace ${workspaceId}`, | ||||
|       ); | ||||
|     } | ||||
|  | ||||
|     if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) { | ||||
|       throw new Error( | ||||
|       throw new TwentyORMException( | ||||
|         `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`, | ||||
|         TwentyORMExceptionCode.METADATA_VERSION_MISMATCH, | ||||
|       ); | ||||
|     } | ||||
|  | ||||
|     const workspaceDataSource = await this.cacheManager.execute( | ||||
|       `${workspaceId}-${latestWorkspaceMetadataVersion}`, | ||||
|       `${workspaceId}-${desiredWorkspaceMetadataVersion}`, | ||||
|       async () => { | ||||
|         let cachedObjectMetadataCollection = | ||||
|         const cachedObjectMetadataCollection = | ||||
|           await this.workspaceCacheStorageService.getObjectMetadataCollection( | ||||
|             workspaceId, | ||||
|             desiredWorkspaceMetadataVersion, | ||||
|           ); | ||||
|  | ||||
|         if (!cachedObjectMetadataCollection) { | ||||
|           const freshObjectMetadataCollection = | ||||
|             await this.objectMetadataRepository.find({ | ||||
|               where: { workspaceId }, | ||||
|               relations: [ | ||||
|                 'fields.object', | ||||
|                 'fields', | ||||
|                 'fields.fromRelationMetadata', | ||||
|                 'fields.toRelationMetadata', | ||||
|                 'fields.fromRelationMetadata.toObjectMetadata', | ||||
|               ], | ||||
|             }); | ||||
|  | ||||
|           await this.workspaceCacheStorageService.setObjectMetadataCollection( | ||||
|           await this.workspaceMetadataCacheService.recomputeMetadataCache( | ||||
|             workspaceId, | ||||
|             freshObjectMetadataCollection, | ||||
|             true, | ||||
|           ); | ||||
|  | ||||
|           cachedObjectMetadataCollection = freshObjectMetadataCollection; | ||||
|           throw new TwentyORMException( | ||||
|             `Object metadata collection not found for workspace ${workspaceId}`, | ||||
|             TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND, | ||||
|           ); | ||||
|         } | ||||
|  | ||||
|         const dataSourceMetadata = | ||||
| @@ -85,20 +80,16 @@ export class WorkspaceDatasourceFactory { | ||||
|           ); | ||||
|  | ||||
|         if (!dataSourceMetadata) { | ||||
|           throw new Error( | ||||
|             `Data source metadata not found for workspace ${workspaceId}`, | ||||
|           ); | ||||
|         } | ||||
|  | ||||
|         if (!cachedObjectMetadataCollection) { | ||||
|           throw new Error( | ||||
|             `Object metadata collection not found for workspace ${workspaceId}`, | ||||
|           throw new TwentyORMException( | ||||
|             `Workspace Schema not found for workspace ${workspaceId}`, | ||||
|             TwentyORMExceptionCode.WORKSPACE_SCHEMA_NOT_FOUND, | ||||
|           ); | ||||
|         } | ||||
|  | ||||
|         const cachedEntitySchemaOptions = | ||||
|           await this.workspaceCacheStorageService.getORMEntitySchema( | ||||
|             workspaceId, | ||||
|             desiredWorkspaceMetadataVersion, | ||||
|           ); | ||||
|  | ||||
|         let cachedEntitySchemas: EntitySchema[]; | ||||
| @@ -110,12 +101,17 @@ export class WorkspaceDatasourceFactory { | ||||
|         } else { | ||||
|           const entitySchemas = await Promise.all( | ||||
|             cachedObjectMetadataCollection.map((objectMetadata) => | ||||
|               this.entitySchemaFactory.create(workspaceId, objectMetadata), | ||||
|               this.entitySchemaFactory.create( | ||||
|                 workspaceId, | ||||
|                 desiredWorkspaceMetadataVersion, | ||||
|                 objectMetadata, | ||||
|               ), | ||||
|             ), | ||||
|           ); | ||||
|  | ||||
|           await this.workspaceCacheStorageService.setORMEntitySchema( | ||||
|             workspaceId, | ||||
|             desiredWorkspaceMetadataVersion, | ||||
|             entitySchemas.map((entitySchema) => entitySchema.options), | ||||
|           ); | ||||
|  | ||||
|   | ||||
| @@ -47,6 +47,10 @@ export class TwentyORMGlobalManager { | ||||
|   } | ||||
|  | ||||
|   async getDataSourceForWorkspace(workspaceId: string) { | ||||
|     return this.workspaceDataSourceFactory.create(workspaceId, null); | ||||
|     return await this.workspaceDataSourceFactory.create(workspaceId, null); | ||||
|   } | ||||
|  | ||||
|   async loadDataSourceForWorkspace(workspaceId: string) { | ||||
|     await this.workspaceDataSourceFactory.create(workspaceId, null); | ||||
|   } | ||||
| } | ||||
|   | ||||
| @@ -3,7 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; | ||||
|  | ||||
| import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; | ||||
| import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; | ||||
| import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; | ||||
| import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; | ||||
| import { entitySchemaFactories } from 'src/engine/twenty-orm/factories'; | ||||
| import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; | ||||
| import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; | ||||
| @@ -16,7 +16,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/ | ||||
|     TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'), | ||||
|     DataSourceModule, | ||||
|     WorkspaceCacheStorageModule, | ||||
|     WorkspaceMetadataVersionModule, | ||||
|     WorkspaceMetadataCacheModule, | ||||
|   ], | ||||
|   providers: [ | ||||
|     ...entitySchemaFactories, | ||||
|   | ||||
| @@ -13,6 +13,7 @@ enum WorkspaceCacheKeys { | ||||
|   GraphQLOperations = 'graphql:operations', | ||||
|   ORMEntitySchemas = 'orm:entity-schemas', | ||||
|   MetadataObjectMetadataCollection = 'metadata:object-metadata-collection', | ||||
|   MetadataObjectMetadataCollectionOngoingCachingLock = 'metadata:object-metadata-collection-ongoing-caching-lock', | ||||
|   MetadataVersion = 'metadata:workspace-metadata-version', | ||||
| } | ||||
|  | ||||
| @@ -25,26 +26,31 @@ export class WorkspaceCacheStorageService { | ||||
|  | ||||
|   setORMEntitySchema( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|     entitySchemas: EntitySchemaOptions<any>[], | ||||
|   ) { | ||||
|     return this.cacheStorageService.set<EntitySchemaOptions<any>[]>( | ||||
|       `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`, | ||||
|       entitySchemas, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   getORMEntitySchema( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|   ): Promise<EntitySchemaOptions<any>[] | undefined> { | ||||
|     return this.cacheStorageService.get<EntitySchemaOptions<any>[]>( | ||||
|       `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   setMetadataVersion(workspaceId: string, version: number): Promise<void> { | ||||
|   setMetadataVersion( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|   ): Promise<void> { | ||||
|     return this.cacheStorageService.set<number>( | ||||
|       `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, | ||||
|       version, | ||||
|       metadataVersion, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
| @@ -54,70 +60,113 @@ export class WorkspaceCacheStorageService { | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   addObjectMetadataCollectionOngoingCachingLock( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|   ) { | ||||
|     return this.cacheStorageService.set<boolean>( | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`, | ||||
|       true, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   removeObjectMetadataCollectionOngoingCachingLock( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|   ) { | ||||
|     return this.cacheStorageService.del( | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   getObjectMetadataCollectionOngoingCachingLock( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|   ): Promise<boolean | undefined> { | ||||
|     return this.cacheStorageService.get<boolean>( | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   setObjectMetadataCollection( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|     objectMetadataCollection: ObjectMetadataEntity[], | ||||
|   ) { | ||||
|     return this.cacheStorageService.set<ObjectMetadataEntity[]>( | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}:${metadataVersion}`, | ||||
|       objectMetadataCollection, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   getObjectMetadataCollection( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|   ): Promise<ObjectMetadataEntity[] | undefined> { | ||||
|     return this.cacheStorageService.get<ObjectMetadataEntity[]>( | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   setGraphQLTypeDefs(workspaceId: string, typeDefs: string): Promise<void> { | ||||
|   setGraphQLTypeDefs( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|     typeDefs: string, | ||||
|   ): Promise<void> { | ||||
|     return this.cacheStorageService.set<string>( | ||||
|       `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`, | ||||
|       typeDefs, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   getGraphQLTypeDefs(workspaceId: string): Promise<string | undefined> { | ||||
|   getGraphQLTypeDefs( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|   ): Promise<string | undefined> { | ||||
|     return this.cacheStorageService.get<string>( | ||||
|       `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   setGraphQLUsedScalarNames( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|     usedScalarNames: string[], | ||||
|   ): Promise<void> { | ||||
|     return this.cacheStorageService.set<string[]>( | ||||
|       `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`, | ||||
|       usedScalarNames, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   getGraphQLUsedScalarNames( | ||||
|     workspaceId: string, | ||||
|     metadataVersion: number, | ||||
|   ): Promise<string[] | undefined> { | ||||
|     return this.cacheStorageService.get<string[]>( | ||||
|       `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|   } | ||||
|  | ||||
|   async flush(workspaceId: string): Promise<void> { | ||||
|   async flush(workspaceId: string, metadataVersion: number): Promise<void> { | ||||
|     await this.cacheStorageService.del( | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|     await this.cacheStorageService.del( | ||||
|       `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|     await this.cacheStorageService.del( | ||||
|       `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|     await this.cacheStorageService.del( | ||||
|       `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|     await this.cacheStorageService.del( | ||||
|       `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, | ||||
|       `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|  | ||||
|     await this.cacheStorageService.del( | ||||
|       `${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`, | ||||
|     ); | ||||
|   } | ||||
| } | ||||
|   | ||||
| @@ -6,7 +6,7 @@ import { DataSource, QueryFailedError } from 'typeorm'; | ||||
| import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; | ||||
|  | ||||
| import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; | ||||
| import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; | ||||
| import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; | ||||
| import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; | ||||
| import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service'; | ||||
|   | ||||
		Reference in New Issue
	
	Block a user