scottk/twenty
1
0
Fork 0
mirror of https://github.com/lingble/twenty.git synced 2025-10-31 12:47:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize sync, reset, seed commands to flush cache and to use less memory (#7034)

Browse Source 
In this PR:
- removing ugprade-0.24 commands as we are releasing 0.30
- introducing cache:flush command
- refactoring upgrade command and sync-metadata command to use the
ActiveWorkspacesCommand so they consistently run on all workspaces or
selected workspaces

Fixes:
- clear localStorage on sign out
- fix missing workspaceMember in verify resolver
- do not throw on datasource already destroyed exception which can
happen with race condition when several resolvers are resolving in
parallel
This commit is contained in:
Charles Bochet
2024-09-15 12:47:45 +02:00
committed by GitHub
parent 0dbd4a7665
commit f54eea0227
31 changed files with 110 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/twenty-front/src/modules/auth/hooks/useAuth.ts
View File

@@ -254,6 +254,7 @@ export const useAuth = () => {
await client.clearStore();
sessionStorage.clear();
localStorage.clear();
},
[client, goToRecoilSnapshot],
);

1
packages/twenty-server/project.json
View File

@@ -149,6 +149,7 @@
"nx ts-node-no-deps -- ./scripts/truncate-db.ts",
"nx ts-node-no-deps -- ./scripts/setup-db.ts",
"nx database:migrate",
"nx command-no-deps -- cache:flush",
"nx command-no-deps -- workspace:seed:dev"
],
"parallel": false

6
packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts
View File

@@ -30,11 +30,11 @@ import { seedPeople } from 'src/database/typeorm-seeds/workspace/people';
import { seedWorkspaceMember } from 'src/database/typeorm-seeds/workspace/workspace-members';
import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';

6
packages/twenty-server/src/database/commands/database-command.module.ts
View File

@@ -7,7 +7,6 @@ import { DataSeedDemoWorkspaceCommand } from 'src/database/commands/data-seed-de
import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module';
import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command';
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
import { UpgradeTo0_24CommandModule } from 'src/database/commands/upgrade-version/0-24/0-24-upgrade-version.module';
import { UpgradeTo0_30CommandModule } from 'src/database/commands/upgrade-version/0-30/0-30-upgrade-version.module';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
@@ -20,9 +19,9 @@ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
@Module({
@@ -41,12 +40,11 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp
WorkspaceModule,
WorkspaceDataSourceModule,
WorkspaceSyncMetadataModule,
WorkspaceStatusModule,
ObjectMetadataModule,
FieldMetadataModule,
DataSeedDemoWorkspaceModule,
WorkspaceCacheStorageModule,
WorkspaceMetadataVersionModule,
UpgradeTo0_24CommandModule,
UpgradeTo0_30CommandModule,
],
providers: [

222
packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command.ts
View File

@@ -1,222 +0,0 @@
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import chalk from 'chalk';
import { Command, CommandRunner, Option } from 'nest-commander';
import { Any, Repository } from 'typeorm';
import {
Workspace,
WorkspaceActivationStatus,
} from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { MessageDirection } from 'src/modules/messaging/common/enums/message-direction.enum';
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
interface SetMessageDirectionCommandOptions {
workspaceId?: string;
}
const MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE = 10;
@Command({
name: 'upgrade-0.24:set-message-direction',
description: 'Set message direction',
})
export class SetMessageDirectionCommand extends CommandRunner {
private readonly logger = new Logger(SetMessageDirectionCommand.name);
constructor(
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
) {
super();
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id. Command runs on all workspaces if not provided',
required: false,
})
parseWorkspaceId(value: string): string {
return value;
}
async run(
_passedParam: string[],
options: SetMessageDirectionCommandOptions,
): Promise<void> {
let activeWorkspaceIds: string[] = [];
if (options.workspaceId) {
activeWorkspaceIds = [options.workspaceId];
} else {
const activeWorkspaces = await this.workspaceRepository.find({
where: {
activationStatus: WorkspaceActivationStatus.ACTIVE,
...(options.workspaceId && { id: options.workspaceId }),
},
});
activeWorkspaceIds = activeWorkspaces.map((workspace) => workspace.id);
}
if (!activeWorkspaceIds.length) {
this.logger.log(chalk.yellow('No workspace found'));
return;
} else {
this.logger.log(
chalk.green(
`Running command on ${activeWorkspaceIds.length} workspaces`,
),
);
}
for (const workspaceId of activeWorkspaceIds) {
try {
const messageChannelMessageAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace(
workspaceId,
);
await workspaceDataSource.transaction(async (transactionManager) => {
try {
const messageChannelMessageAssociationCount =
await messageChannelMessageAssociationRepository.count(
{},
transactionManager,
);
for (
let i = 0;
i < messageChannelMessageAssociationCount;
i += MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE
) {
const messageChannelMessageAssociationsPage =
await messageChannelMessageAssociationRepository.find(
{
where: {
message: {
messageParticipants: {
role: 'from',
},
},
},
relations: {
message: {
messageParticipants: true,
},
messageChannel: {
connectedAccount: true,
},
},
take: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE,
skip: i,
},
transactionManager,
);
const { incoming, outgoing } =
messageChannelMessageAssociationsPage.reduce(
(
acc: {
incoming: string[];
outgoing: string[];
},
messageChannelMessageAssociation,
) => {
const connectedAccountHandle =
messageChannelMessageAssociation?.messageChannel
?.connectedAccount?.handle;
const connectedAccountHandleAliases =
messageChannelMessageAssociation?.messageChannel
?.connectedAccount?.handleAliases;
const fromHandle =
messageChannelMessageAssociation?.message
?.messageParticipants?.[0]?.handle ?? '';
if (
connectedAccountHandle === fromHandle ||
connectedAccountHandleAliases?.includes(fromHandle)
) {
acc.outgoing.push(messageChannelMessageAssociation.id);
} else {
acc.incoming.push(messageChannelMessageAssociation.id);
}
return acc;
},
{ incoming: [], outgoing: [] },
);
await messageChannelMessageAssociationRepository.update(
{
id: Any(incoming),
},
{
direction: MessageDirection.INCOMING,
},
transactionManager,
);
await messageChannelMessageAssociationRepository.update(
{
id: Any(outgoing),
},
{
direction: MessageDirection.OUTGOING,
},
transactionManager,
);
const numberOfProcessedAssociations =
i + MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE;
if (
numberOfProcessedAssociations %
(MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE * 10) ===
0 ||
numberOfProcessedAssociations >=
messageChannelMessageAssociationCount
) {
this.logger.log(
chalk.green(
`Processed ${Math.min(numberOfProcessedAssociations, messageChannelMessageAssociationCount)} of ${messageChannelMessageAssociationCount} message channel message associations`,
),
);
}
}
} catch (error) {
this.logger.log(
chalk.red(`Running command on workspace ${workspaceId} failed`),
);
throw error;
}
});
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
this.logger.log(
chalk.green(`Running command on workspace ${workspaceId} done`),
);
} catch (error) {
this.logger.error(
`Migration failed for workspace ${workspaceId}: ${error.message}`,
);
}
}
this.logger.log(chalk.green(`Command completed!`));
}
}

42
packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-upgrade-version.command.ts
View File

@@ -1,42 +0,0 @@
import { Command, CommandRunner, Option } from 'nest-commander';
import { SetMessageDirectionCommand } from 'src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command';
import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command';
interface UpdateTo0_24CommandOptions {
workspaceId?: string;
}
@Command({
name: 'upgrade-0.24',
description: 'Upgrade to 0.24',
})
export class UpgradeTo0_24Command extends CommandRunner {
constructor(
private readonly syncWorkspaceMetadataCommand: SyncWorkspaceMetadataCommand,
private readonly setMessagesDirectionCommand: SetMessageDirectionCommand,
) {
super();
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description:
'workspace id. Command runs on all active workspaces if not provided',
required: false,
})
parseWorkspaceId(value: string): string {
return value;
}
async run(
passedParam: string[],
options: UpdateTo0_24CommandOptions,
): Promise<void> {
await this.syncWorkspaceMetadataCommand.run(passedParam, {
...options,
force: true,
});
await this.setMessagesDirectionCommand.run(passedParam, options);
}
}

43
packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-upgrade-version.module.ts
View File

@@ -1,43 +0,0 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { SetMessageDirectionCommand } from 'src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command';
import { UpgradeTo0_24Command } from 'src/database/commands/upgrade-version/0-24/0-24-upgrade-version.command';
import { SetCustomObjectIsSoftDeletableCommand } from 'src/database/commands/upgrade-version/0-30/0-30-set-custom-object-is-soft-deletable.command';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity';
import { OnboardingModule } from 'src/engine/core-modules/onboarding/onboarding.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module';
@Module({
imports: [
TypeOrmModule.forFeature([Workspace, KeyValuePair], 'core'),
WorkspaceSyncMetadataCommandsModule,
FileStorageModule,
OnboardingModule,
TypeORMModule,
DataSourceModule,
WorkspaceMetadataVersionModule,
FieldMetadataModule,
WorkspaceStatusModule,
TypeOrmModule.forFeature(
[FieldMetadataEntity, ObjectMetadataEntity],
'metadata',
),
TypeORMModule,
],
providers: [
UpgradeTo0_24Command,
SetMessageDirectionCommand,
SetCustomObjectIsSoftDeletableCommand,
],
})
export class UpgradeTo0_24CommandModule {}

46
packages/twenty-server/src/database/commands/upgrade-version/0-30/0-30-upgrade-version.command.ts
View File

@@ -1,7 +1,12 @@
import { Command, CommandRunner, Option } from 'nest-commander';
import { InjectRepository } from '@nestjs/typeorm';
import { Command } from 'nest-commander';
import { Repository } from 'typeorm';
import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command';
import { MigrateEmailFieldsToEmailsCommand } from 'src/database/commands/upgrade-version/0-30/0-30-migrate-email-fields-to-emails.command';
import { SetCustomObjectIsSoftDeletableCommand } from 'src/database/commands/upgrade-version/0-30/0-30-set-custom-object-is-soft-deletable.command';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command';
interface UpdateTo0_30CommandOptions {
@@ -12,34 +17,39 @@ interface UpdateTo0_30CommandOptions {
name: 'upgrade-0.30',
description: 'Upgrade to 0.30',
})
export class UpgradeTo0_30Command extends CommandRunner {
export class UpgradeTo0_30Command extends ActiveWorkspacesCommandRunner {
constructor(
@InjectRepository(Workspace, 'core')
protected readonly workspaceRepository: Repository<Workspace>,
private readonly syncWorkspaceMetadataCommand: SyncWorkspaceMetadataCommand,
private readonly migrateEmailFieldsToEmails: MigrateEmailFieldsToEmailsCommand,
private readonly setCustomObjectIsSoftDeletableCommand: SetCustomObjectIsSoftDeletableCommand,
) {
super();
super(workspaceRepository);
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description:
'workspace id. Command runs on all active workspaces if not provided',
required: false,
})
parseWorkspaceId(value: string): string {
return value;
}
async run(
async executeActiveWorkspacesCommand(
passedParam: string[],
options: UpdateTo0_30CommandOptions,
workspaceIds: string[],
): Promise<void> {
await this.syncWorkspaceMetadataCommand.run(passedParam, {
await this.syncWorkspaceMetadataCommand.executeActiveWorkspacesCommand(
passedParam,
{
...options,
force: true,
});
await this.setCustomObjectIsSoftDeletableCommand.run(passedParam, options);
await this.migrateEmailFieldsToEmails.run(passedParam, options);
},
workspaceIds,
);
await this.setCustomObjectIsSoftDeletableCommand.executeActiveWorkspacesCommand(
passedParam,
options,
workspaceIds,
);
await this.migrateEmailFieldsToEmails.executeActiveWorkspacesCommand(
passedParam,
options,
workspaceIds,
);
}
}

2
packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts
View File

@@ -4,7 +4,7 @@ import GraphQLJSON from 'graphql-type-json';
import { useCachedMetadata } from 'src/engine/api/graphql/graphql-config/hooks/use-cached-metadata';
import { useThrottler } from 'src/engine/api/graphql/graphql-config/hooks/use-throttler';
import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphql-api.module';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { useGraphQLErrorHandlerHook } from 'src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook';

9
packages/twenty-server/src/engine/core-modules/auth/services/auth.service.ts
View File

@@ -34,7 +34,6 @@ import { WorkspaceInviteHashValid } from 'src/engine/core-modules/auth/dto/works
import { SignInUpService } from 'src/engine/core-modules/auth/services/sign-in-up.service';
import { EmailService } from 'src/engine/core-modules/email/email.service';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { WorkspaceMember } from 'src/engine/core-modules/user/dtos/workspace-member.dto';
import { UserService } from 'src/engine/core-modules/user/services/user.service';
import { User } from 'src/engine/core-modules/user/user.entity';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
@@ -150,14 +149,6 @@ export class AuthService {
// passwordHash is hidden for security reasons
user.passwordHash = '';
const workspaceMember = await this.userService.loadWorkspaceMember(
user,
user.defaultWorkspace,
);
if (workspaceMember) {
user.workspaceMember = workspaceMember as WorkspaceMember;
}
const accessToken = await this.tokenService.generateAccessToken(user.id);
const refreshToken = await this.tokenService.generateRefreshToken(user.id);

12
packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module.ts
View File

@@ -1,11 +1,12 @@
import { Module, Global, Inject, OnModuleDestroy } from '@nestjs/common';
import { CacheModule, CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
import { CACHE_MANAGER, Cache, CacheModule } from '@nestjs/cache-manager';
import { Global, Inject, Module, OnModuleDestroy } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { cacheStorageModuleFactory } from 'src/engine/core-modules/cache-storage/cache-storage.module-factory';
import { FlushCacheCommand } from 'src/engine/core-modules/cache-storage/commands/flush-cache.command';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
@Global()
@Module({
@@ -25,8 +26,9 @@ import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/typ
},
inject: [CACHE_MANAGER],
})),
FlushCacheCommand,
],
exports: [...Object.values(CacheStorageNamespace)],
exports: [...Object.values(CacheStorageNamespace), FlushCacheCommand],
})
export class CacheStorageModule implements OnModuleDestroy {
constructor(@Inject(CACHE_MANAGER) private cacheManager: Cache) {}

29
packages/twenty-server/src/engine/core-modules/cache-storage/commands/flush-cache.command.ts Normal file
View File

@@ -0,0 +1,29 @@
import { Logger } from '@nestjs/common';
import { Command, CommandRunner } from 'nest-commander';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
// TODO: implement dry-run
@Command({
name: 'cache:flush',
description: 'Completely flush cache',
})
export class FlushCacheCommand extends CommandRunner {
private readonly logger = new Logger(FlushCacheCommand.name);
constructor(
@InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
private readonly cacheStorage: CacheStorageService,
) {
super();
}
async run(): Promise<void> {
this.logger.log('Flushing cache...');
await this.cacheStorage.flush();
this.logger.log('Cache flushed');
}
}

0
packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.service.ts → packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts
View File

6
packages/twenty-server/src/engine/core-modules/throttler/throttler.service.ts
View File

@@ -1,12 +1,12 @@
import { Injectable } from '@nestjs/common';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import {
ThrottlerException,
ThrottlerExceptionCode,
} from 'src/engine/core-modules/throttler/throttler.exception';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
@Injectable()
export class ThrottlerService {

2
packages/twenty-server/src/engine/core-modules/user/user.resolver.ts
View File

@@ -103,7 +103,7 @@ export class UserResolver {
): Promise<WorkspaceMember | null> {
const workspaceMember = await this.userService.loadWorkspaceMember(
user,
workspace,
workspace ?? user.defaultWorkspace,
);
if (workspaceMember && workspaceMember.avatarUrl) {

2
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts
View File

@@ -21,7 +21,6 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { FieldMetadataEntity } from './field-metadata.entity';
import { FieldMetadataService } from './field-metadata.service';
@@ -38,7 +37,6 @@ import { UpdateFieldInput } from './dtos/update-field.input';
'metadata',
),
WorkspaceMigrationModule,
WorkspaceStatusModule,
WorkspaceMigrationRunnerModule,
WorkspaceMetadataVersionModule,
ObjectMetadataModule,

2
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts
View File

@@ -48,7 +48,5 @@ export class WorkspaceMetadataVersionService {
await this.workspaceMetadataCacheService.recomputeMetadataCache(
workspaceId,
);
await this.twentyORMGlobalManager.loadDataSourceForWorkspace(workspaceId);
}
}

13
packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
View File

@@ -146,7 +146,18 @@ export class WorkspaceDatasourceFactory {
return workspaceDataSource;
},
(dataSource) => dataSource.destroy(),
async (dataSource) => {
try {
await dataSource.destroy();
} catch (error) {
// Ignore error if pool has already been destroyed which is a common race condition case
if (error.message === 'Called end on pool more than once') {
return;
}
throw error;
}
},
);
if (!workspaceDataSource) {

2
packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts
View File

@@ -2,8 +2,8 @@ import { Injectable } from '@nestjs/common';
import { EntitySchemaOptions } from 'typeorm';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';

2
packages/twenty-server/src/engine/workspace-manager/workspace-manager.module.ts
View File

@@ -5,7 +5,6 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
import { WorkspaceManagerService } from './workspace-manager.service';
@@ -18,7 +17,6 @@ import { WorkspaceManagerService } from './workspace-manager.service';
DataSourceModule,
WorkspaceSyncMetadataModule,
WorkspaceHealthModule,
WorkspaceStatusModule,
],
exports: [WorkspaceManagerService],
providers: [WorkspaceManagerService],

72
packages/twenty-server/src/engine/workspace-manager/workspace-status/services/workspace-status.service.ts
View File

@@ -1,72 +0,0 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Any, Repository } from 'typeorm';
import {
BillingSubscription,
SubscriptionStatus,
} from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
@Injectable()
export class WorkspaceStatusService {
constructor(
private readonly environmentService: EnvironmentService,
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
@InjectRepository(BillingSubscription, 'core')
private readonly billingSubscriptionRepository: Repository<BillingSubscription>,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
) {}
async getActiveWorkspaceIds(): Promise<string[]> {
const workspaces = await this.workspaceRepository.find();
const workspaceIds = workspaces.map((workspace) => workspace.id);
if (!this.environmentService.get('IS_BILLING_ENABLED')) {
return workspaceIds;
}
const billingSubscriptionForWorkspaces =
await this.billingSubscriptionRepository.find({
where: {
workspaceId: Any(workspaceIds),
status: Any([
SubscriptionStatus.PastDue,
SubscriptionStatus.Active,
SubscriptionStatus.Trialing,
]),
},
});
const workspaceIdsWithActiveSubscription =
billingSubscriptionForWorkspaces.map(
(billingSubscription) => billingSubscription.workspaceId,
);
const freeAccessEnabledFeatureFlagForWorkspace =
await this.featureFlagRepository.find({
where: {
workspaceId: Any(workspaceIds),
key: FeatureFlagKey.IsFreeAccessEnabled,
value: true,
},
});
const workspaceIdsWithFreeAccessEnabled =
freeAccessEnabledFeatureFlagForWorkspace.map(
(featureFlag) => featureFlag.workspaceId,
);
return workspaceIds.filter(
(workspaceId) =>
workspaceIdsWithActiveSubscription.includes(workspaceId) ||
workspaceIdsWithFreeAccessEnabled.includes(workspaceId),
);
}
}

21
packages/twenty-server/src/engine/workspace-manager/workspace-status/workspace-manager.module.ts
View File

@@ -1,21 +0,0 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module';
import { WorkspaceStatusService } from 'src/engine/workspace-manager/workspace-status/services/workspace-status.service';
@Module({
imports: [
EnvironmentModule,
TypeOrmModule.forFeature(
[Workspace, BillingSubscription, FeatureFlagEntity],
'core',
),
],
exports: [WorkspaceStatusService],
providers: [WorkspaceStatusService],
})
export class WorkspaceStatusModule {}

43
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts
View File

@@ -1,11 +1,12 @@
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import isEmpty from 'lodash.isempty';
import { Command, CommandRunner, Option } from 'nest-commander';
import { Command, Option } from 'nest-commander';
import { Repository } from 'typeorm';
import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
import { WorkspaceStatusService } from 'src/engine/workspace-manager/workspace-status/services/workspace-status.service';
import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service';
import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service';
@@ -21,35 +22,24 @@ interface RunWorkspaceMigrationsOptions {
name: 'workspace:sync-metadata',
description: 'Sync metadata',
})
export class SyncWorkspaceMetadataCommand extends CommandRunner {
private readonly logger = new Logger(SyncWorkspaceMetadataCommand.name);
export class SyncWorkspaceMetadataCommand extends ActiveWorkspacesCommandRunner {
constructor(
@InjectRepository(Workspace, 'core')
protected readonly workspaceRepository: Repository<Workspace>,
private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService,
private readonly workspaceHealthService: WorkspaceHealthService,
private readonly dataSourceService: DataSourceService,
private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService,
private readonly workspaceStatusService: WorkspaceStatusService,
) {
super();
super(workspaceRepository);
}
async run(
async executeActiveWorkspacesCommand(
_passedParam: string[],
options: RunWorkspaceMigrationsOptions,
workspaceIds: string[],
): Promise<void> {
// TODO: re-implement load index from workspaceService, this is breaking the logger
let workspaceIds = options.workspaceId ? [options.workspaceId] : [];
if (isEmpty(workspaceIds)) {
const activeWorkspaceIds =
await this.workspaceStatusService.getActiveWorkspaceIds();
workspaceIds = activeWorkspaceIds;
this.logger.log(
`Attempting to sync ${activeWorkspaceIds.length} workspaces.`,
);
}
this.logger.log(`Attempting to sync ${workspaceIds.length} workspaces.`);
let count = 1;
@@ -138,15 +128,6 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
);
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id',
required: false,
})
parseWorkspaceId(value: string): string {
return value;
}
@Option({
flags: '-d, --dry-run',
description: 'Dry run without applying changes',

2
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts
View File

@@ -6,7 +6,6 @@ import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.mod
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { ConvertRecordPositionsToIntegers } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/convert-record-positions-to-integers.command';
import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
@@ -22,7 +21,6 @@ import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.ser
DataSourceModule,
WorkspaceDataSourceModule,
TypeOrmModule.forFeature([Workspace], 'core'),
WorkspaceStatusModule,
],
providers: [
SyncWorkspaceMetadataCommand,

2
packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts
View File

@@ -2,8 +2,8 @@ import { Injectable } from '@nestjs/common';
import { Any } from 'typeorm';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import {

2
packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts
View File

@@ -2,8 +2,8 @@ import { Injectable } from '@nestjs/common';
import { Any } from 'typeorm';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';

2
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts
View File

@@ -1,5 +1,5 @@
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';

2
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts
View File

@@ -1,7 +1,7 @@
import { Logger } from '@nestjs/common';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';

2
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts
View File

@@ -2,8 +2,8 @@ import { Injectable } from '@nestjs/common';
import { Any } from 'typeorm';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';

2
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts
View File

@@ -1,7 +1,7 @@
import { Injectable, Logger } from '@nestjs/common';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';

2
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts
View File

@@ -2,8 +2,8 @@ import { Injectable, Logger } from '@nestjs/common';
import { Any } from 'typeorm';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';