From 4f9527c8605bcba6f021c651492fd7e1ee3945cd Mon Sep 17 00:00:00 2001 From: bosiraphael <71827178+bosiraphael@users.noreply.github.com> Date: Thu, 27 Jun 2024 16:37:34 +0200 Subject: [PATCH] 5901 refactor email and calendar auto contact creation to create them by batch (#6038) Closes #5901 --- package.json | 2 + ...-companies-and-contacts-creation.module.ts | 6 +- .../contacts-creation-batch-size.constant.ts | 1 + ...acts-creation-message-channel.listener.ts} | 4 +- .../create-company-and-contact.service.ts | 89 ++++++++++--------- .../types/contact.type.ts | 2 - .../get-unique-contacts-and-handles.spec.ts | 6 +- ...contacts-from-company-or-workspace.util.ts | 6 +- .../get-unique-contacts-and-handles.util.ts | 6 +- yarn.lock | 18 ++++ 10 files changed, 86 insertions(+), 54 deletions(-) create mode 100644 packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant.ts rename packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/{messaging-message-channel.listener.ts => auto-companies-and-contacts-creation-message-channel.listener.ts} (93%) diff --git a/package.json b/package.json index bf61c2d2b..42d37ebab 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "@types/dompurify": "^3.0.5", "@types/facepaint": "^1.2.5", "@types/lodash.camelcase": "^4.3.7", + "@types/lodash.chunk": "^4.2.9", "@types/lodash.merge": "^4.6.7", "@types/lodash.pick": "^4.3.7", "@types/nodemailer": "^6.4.14", @@ -114,6 +115,7 @@ "jsonwebtoken": "^9.0.0", "libphonenumber-js": "^1.10.26", "lodash.camelcase": "^4.3.0", + "lodash.chunk": "^4.2.0", "lodash.compact": "^3.0.1", "lodash.debounce": "^4.0.8", "lodash.groupby": "^4.6.0", diff --git a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module.ts b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module.ts index 15a6131d1..b46d31df2 100644 --- a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module.ts +++ b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module.ts @@ -11,6 +11,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works import { CalendarEventParticipantModule } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.module'; import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module'; +import { AutoCompaniesAndContactsCreationMessageChannelListener } from 'src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener'; @Module({ imports: [ @@ -25,7 +26,10 @@ import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-co CalendarEventParticipantModule, TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), ], - providers: [CreateCompanyAndContactService], + providers: [ + CreateCompanyAndContactService, + AutoCompaniesAndContactsCreationMessageChannelListener, + ], exports: [CreateCompanyAndContactService], }) export class AutoCompaniesAndContactsCreationModule {} diff --git a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant.ts b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant.ts new file mode 100644 index 000000000..3e96e5317 --- /dev/null +++ b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant.ts @@ -0,0 +1 @@ +export const CONTACTS_CREATION_BATCH_SIZE = 100; diff --git a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/messaging-message-channel.listener.ts b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts similarity index 93% rename from packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/messaging-message-channel.listener.ts rename to packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts index 01a047653..66fb9376c 100644 --- a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/messaging-message-channel.listener.ts +++ b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts @@ -13,9 +13,9 @@ import { } from 'src/modules/messaging/message-participants-manager/jobs/messaging-create-company-and-contact-after-sync.job'; @Injectable() -export class MessagingMessageChannelListener { +export class AutoCompaniesAndContactsCreationMessageChannelListener { constructor( - @InjectMessageQueue(MessageQueue.messagingQueue) + @InjectMessageQueue(MessageQueue.contactCreationQueue) private readonly messageQueueService: MessageQueueService, ) {} diff --git a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts index 3238c094a..153904526 100644 --- a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts +++ b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts @@ -3,6 +3,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { EntityManager } from 'typeorm'; import compact from 'lodash.compact'; +import chunk from 'lodash.chunk'; import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util'; import { CreateCompanyService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-company/create-company.service'; @@ -14,7 +15,6 @@ import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repos import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util'; -import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type'; import { CalendarEventParticipantService } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.service'; import { filterOutContactsFromCompanyOrWorkspace } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util'; import { MessagingMessageParticipantService } from 'src/modules/messaging/common/services/messaging-message-participant.service'; @@ -23,6 +23,8 @@ import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/st import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource'; import { InjectWorkspaceDatasource } from 'src/engine/twenty-orm/decorators/inject-workspace-datasource.decorator'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant'; +import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type'; @Injectable() export class CreateCompanyAndContactService { @@ -42,7 +44,7 @@ export class CreateCompanyAndContactService { async createCompaniesAndPeople( connectedAccountHandle: string, - contactsToCreate: Contacts, + contactsToCreate: Contact[], workspaceId: string, transactionManager?: EntityManager, ): Promise { @@ -132,48 +134,55 @@ export class CreateCompanyAndContactService { async createCompaniesAndContactsAndUpdateParticipants( connectedAccount: ConnectedAccountWorkspaceEntity, - contactsToCreate: Contacts, + contactsToCreate: Contact[], workspaceId: string, ) { - let updatedMessageParticipants: MessageParticipantWorkspaceEntity[] = []; - let updatedCalendarEventParticipants: CalendarEventParticipantWorkspaceEntity[] = - []; - - await this.workspaceDataSource?.transaction( - async (transactionManager: EntityManager) => { - const createdPeople = await this.createCompaniesAndPeople( - connectedAccount.handle, - contactsToCreate, - workspaceId, - transactionManager, - ); - - updatedMessageParticipants = - await this.messageParticipantService.updateMessageParticipantsAfterPeopleCreation( - createdPeople, - workspaceId, - transactionManager, - ); - - updatedCalendarEventParticipants = - await this.calendarEventParticipantService.updateCalendarEventParticipantsAfterPeopleCreation( - createdPeople, - workspaceId, - transactionManager, - ); - }, + const contactsBatches = chunk( + contactsToCreate, + CONTACTS_CREATION_BATCH_SIZE, ); - this.eventEmitter.emit(`messageParticipant.matched`, { - workspaceId, - workspaceMemberId: connectedAccount.accountOwnerId, - messageParticipants: updatedMessageParticipants, - }); + for (const contactsBatch of contactsBatches) { + let updatedMessageParticipants: MessageParticipantWorkspaceEntity[] = []; + let updatedCalendarEventParticipants: CalendarEventParticipantWorkspaceEntity[] = + []; - this.eventEmitter.emit(`calendarEventParticipant.matched`, { - workspaceId, - workspaceMemberId: connectedAccount.accountOwnerId, - calendarEventParticipants: updatedCalendarEventParticipants, - }); + await this.workspaceDataSource?.transaction( + async (transactionManager: EntityManager) => { + const createdPeople = await this.createCompaniesAndPeople( + connectedAccount.handle, + contactsBatch, + workspaceId, + transactionManager, + ); + + updatedMessageParticipants = + await this.messageParticipantService.updateMessageParticipantsAfterPeopleCreation( + createdPeople, + workspaceId, + transactionManager, + ); + + updatedCalendarEventParticipants = + await this.calendarEventParticipantService.updateCalendarEventParticipantsAfterPeopleCreation( + createdPeople, + workspaceId, + transactionManager, + ); + }, + ); + + this.eventEmitter.emit(`messageParticipant.matched`, { + workspaceId, + workspaceMemberId: connectedAccount.accountOwnerId, + messageParticipants: updatedMessageParticipants, + }); + + this.eventEmitter.emit(`calendarEventParticipant.matched`, { + workspaceId, + workspaceMemberId: connectedAccount.accountOwnerId, + calendarEventParticipants: updatedCalendarEventParticipants, + }); + } } } diff --git a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type.ts b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type.ts index 0cd41a5b7..404243113 100644 --- a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type.ts +++ b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type.ts @@ -2,5 +2,3 @@ export type Contact = { handle: string; displayName: string; }; - -export type Contacts = Contact[]; diff --git a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/__tests__/get-unique-contacts-and-handles.spec.ts b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/__tests__/get-unique-contacts-and-handles.spec.ts index 87537a8e8..49680a6fc 100644 --- a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/__tests__/get-unique-contacts-and-handles.spec.ts +++ b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/__tests__/get-unique-contacts-and-handles.spec.ts @@ -1,9 +1,9 @@ -import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type'; +import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type'; import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util'; describe('getUniqueContactsAndHandles', () => { it('should return empty arrays when contacts is empty', () => { - const contacts: Contacts = []; + const contacts: Contact[] = []; const result = getUniqueContactsAndHandles(contacts); expect(result.uniqueContacts).toEqual([]); @@ -11,7 +11,7 @@ describe('getUniqueContactsAndHandles', () => { }); it('should return unique contacts and handles', () => { - const contacts: Contacts = [ + const contacts: Contact[] = [ { handle: 'john@twenty.com', displayName: 'John Doe' }, { handle: 'john@twenty.com', displayName: 'John Doe' }, { handle: 'jane@twenty.com', displayName: 'Jane Smith' }, diff --git a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util.ts b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util.ts index c746dabae..23830c6cf 100644 --- a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util.ts +++ b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util.ts @@ -1,12 +1,12 @@ import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; -import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type'; +import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type'; export function filterOutContactsFromCompanyOrWorkspace( - contacts: Contacts, + contacts: Contact[], selfHandle: string, workspaceMembers: WorkspaceMemberWorkspaceEntity[], -): Contacts { +): Contact[] { const selfDomainName = getDomainNameFromHandle(selfHandle); const workspaceMembersMap = workspaceMembers.reduce( diff --git a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util.ts b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util.ts index a6296e3d9..08577612f 100644 --- a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util.ts +++ b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util.ts @@ -1,10 +1,10 @@ import uniq from 'lodash.uniq'; import uniqBy from 'lodash.uniqby'; -import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type'; +import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type'; -export function getUniqueContactsAndHandles(contacts: Contacts): { - uniqueContacts: Contacts; +export function getUniqueContactsAndHandles(contacts: Contact[]): { + uniqueContacts: Contact[]; uniqueHandles: string[]; } { if (contacts.length === 0) { diff --git a/yarn.lock b/yarn.lock index 8c0943b0f..5188200ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17265,6 +17265,15 @@ __metadata: languageName: node linkType: hard +"@types/lodash.chunk@npm:^4.2.9": + version: 4.2.9 + resolution: "@types/lodash.chunk@npm:4.2.9" + dependencies: + "@types/lodash": "npm:*" + checksum: 5759b3d969c5db4b0893b70261ae40d4b9a6466c984c16de6fa1d3945b3199cc09f948a444a3b4e6cfa0dd984044cf937cbc8dab5fe0ac8da67244ed74d9e4e4 + languageName: node + linkType: hard + "@types/lodash.compact@npm:^3.0.9": version: 3.0.9 resolution: "@types/lodash.compact@npm:3.0.9" @@ -34530,6 +34539,13 @@ __metadata: languageName: node linkType: hard +"lodash.chunk@npm:^4.2.0": + version: 4.2.0 + resolution: "lodash.chunk@npm:4.2.0" + checksum: f9f99969561ad2f62af1f9a96c5bd0af776f000292b0d8db3126c28eb3b32e210d7c31b49c18d0d7901869bd769057046dc134b60cfa0c2c4ce017823a26bb23 + languageName: node + linkType: hard + "lodash.clonedeep@npm:^4.5.0": version: 4.5.0 resolution: "lodash.clonedeep@npm:4.5.0" @@ -47351,6 +47367,7 @@ __metadata: "@types/jest": "npm:^29.5.11" "@types/js-cookie": "npm:^3.0.3" "@types/lodash.camelcase": "npm:^4.3.7" + "@types/lodash.chunk": "npm:^4.2.9" "@types/lodash.compact": "npm:^3.0.9" "@types/lodash.debounce": "npm:^4.0.7" "@types/lodash.groupby": "npm:^4.6.9" @@ -47462,6 +47479,7 @@ __metadata: jsonwebtoken: "npm:^9.0.0" libphonenumber-js: "npm:^1.10.26" lodash.camelcase: "npm:^4.3.0" + lodash.chunk: "npm:^4.2.0" lodash.compact: "npm:^3.0.1" lodash.debounce: "npm:^4.0.8" lodash.groupby: "npm:^4.6.0"