0.2.0 cleaning script (#2342)

* Display maxUpdatedAt for each workspace Schema

* Factorize functions

* Add max update for public workspaces

* Merge everything in a single json

* Enrich results

* Get from proper table

* Update

* Move to proper command file

* Add a dry-run option

* Remove workspaces from database

* Fix DeleteWorkspace method

* Add new option

* Remove proper data when deleting workspace

* Minor improvements

server/scripts/setup-db.ts

@@ -1,28 +1,6 @@
-import { ConfigService } from '@nestjs/config';
-
 import console from 'console';
 
-import { config } from 'dotenv';
+import { connectionSource, performQuery } from './utils';
-import { DataSource } from 'typeorm';
-
-config();
-
-const configService = new ConfigService();
-
-export const connectionSource = new DataSource({
-  type: 'postgres',
-  logging: false,
-  url: configService.get<string>('PG_DATABASE_URL'),
-});
-
-const performQuery = async (query: string, consoleDescription: string) => {
-  try {
-    await connectionSource.query(query);
-    console.log(`Performed '${consoleDescription}' successfully`);
-  } catch (err) {
-    console.error(`Failed to perform '${consoleDescription}':`, err);
-  }
-};
 
 connectionSource
   .initialize()

server/scripts/truncate-db.ts

@@ -1,28 +1,6 @@
-import { ConfigService } from '@nestjs/config';
-
 import console from 'console';
 
-import { config } from 'dotenv';
+import { connectionSource, performQuery } from './utils';
-import { DataSource } from 'typeorm';
-
-config();
-
-const configService = new ConfigService();
-
-export const connectionSource = new DataSource({
-  type: 'postgres',
-  logging: false,
-  url: configService.get<string>('PG_DATABASE_URL'),
-});
-
-const performQuery = async (query: string, consoleDescription: string) => {
-  try {
-    await connectionSource.query(query);
-    console.log(`Performed '${consoleDescription}' successfully`);
-  } catch (err) {
-    console.error(`Failed to perform '${consoleDescription}':`, err);
-  }
-};
 
 connectionSource
   .initialize()

server/scripts/utils.ts

@@ -0,0 +1,28 @@
+import { ConfigService } from '@nestjs/config';
+
+import console from 'console';
+
+import { config } from 'dotenv';
+import { DataSource } from 'typeorm';
+
+config();
+const configService = new ConfigService();
+export const connectionSource = new DataSource({
+  type: 'postgres',
+  logging: false,
+  url: configService.get<string>('PG_DATABASE_URL'),
+});
+
+export const performQuery = async (
+  query: string,
+  consoleDescription: string,
+  withLog = true,
+) => {
+  try {
+    const result = await connectionSource.query(query);
+    withLog && console.log(`Performed '${consoleDescription}' successfully`);
+    return result;
+  } catch (err) {
+    withLog && console.error(`Failed to perform '${consoleDescription}':`, err);
+  }
+};

server/src/command.module.ts

@@ -1,10 +1,12 @@
 import { Module } from '@nestjs/common';
 
+import { DatabaseCommandModule } from 'src/database/commands/database-command.module';
+
 import { AppModule } from './app.module';
 
 import { MetadataCommandModule } from './metadata/commands/metadata-command.module';
 
 @Module({
-  imports: [AppModule, MetadataCommandModule],
+  imports: [AppModule, MetadataCommandModule, DatabaseCommandModule],
 })
 export class CommandModule {}

server/src/core/user/user.service.ts

@@ -137,8 +137,6 @@ export class UserService {
       // Delete entire workspace
       await this.workspaceService.deleteWorkspace({
         workspaceId,
-        userId,
-        select: { id: true },
       });
     } else {
       await this.prismaService.client.$transaction([

server/src/core/workspace/resolvers/workspace.resolver.ts

@@ -24,8 +24,6 @@ import {
   UpdateWorkspaceAbilityHandler,
   DeleteWorkspaceAbilityHandler,
 } from 'src/ability/handlers/workspace.ability-handler';
-import { AuthUser } from 'src/decorators/auth-user.decorator';
-import { User } from 'src/core/@generated/user/user.model';
 
 @UseGuards(JwtAuthGuard)
 @Resolver(() => Workspace)
@@ -108,12 +106,10 @@ export class WorkspaceResolver {
     @AuthWorkspace() { id: workspaceId }: Workspace,
     @PrismaSelector({ modelName: 'Workspace' })
     { value: select }: PrismaSelect<'Workspace'>,
-    @AuthUser() { id: userId }: User,
   ) {
     return this.workspaceService.deleteWorkspace({
       workspaceId,
       select,
-      userId,
     });
   }
 }

server/src/core/workspace/services/workspace.service.ts

@@ -93,12 +93,10 @@ export class WorkspaceService {
 
   async deleteWorkspace({
     workspaceId,
-    select,
+    select = { id: true },
-    userId,
   }: {
     workspaceId: string;
-    select: Prisma.WorkspaceSelect;
+    select?: Prisma.WorkspaceSelect;
-    userId: string;
   }) {
     const workspace = await this.findUnique({
       where: { id: workspaceId },
@@ -109,19 +107,17 @@ export class WorkspaceService {
     const where = { workspaceId };
 
     const {
-      user,
       workspaceMember,
-      refreshToken,
       attachment,
       comment,
       activityTarget,
       activity,
+      apiKey,
+      favorite,
+      webHook,
     } = this.prismaService.client;
 
-    const activitys = await activity.findMany({
+    // We don't delete user or refresh tokens as they can belong to another workspace
-      where: { authorId: userId },
-    });
-
     await this.prismaService.client.$transaction([
       this.pipelineProgressService.deleteMany({
         where,
@@ -147,22 +143,20 @@ export class WorkspaceService {
       comment.deleteMany({
         where,
       }),
-      ...activitys.map(({ id: activityId }) =>
+      activityTarget.deleteMany({
-        activityTarget.deleteMany({
+        where,
-          where: { activityId },
+      }),
-        }),
-      ),
       activity.deleteMany({
         where,
       }),
-      refreshToken.deleteMany({
+      apiKey.deleteMany({
-        where: { userId },
+        where,
       }),
-      // Todo delete all users from this workspace
+      favorite.deleteMany({
-      user.delete({
+        where,
-        where: {
+      }),
-          id: userId,
+      webHook.deleteMany({
-        },
+        where,
       }),
       this.delete({ where: { id: workspaceId } }),
     ]);

server/src/database/commands/clean-inactive-workspaces.command.ts

@@ -0,0 +1,237 @@
+import {
+  Command,
+  CommandRunner,
+  InquirerService,
+  Option,
+} from 'nest-commander';
+
+import { PrismaService } from 'src/database/prisma.service';
+import peopleSeed from 'src/core/person/seed-data/people.json';
+import companiesSeed from 'src/core/company/seed-data/companies.json';
+import pipelineStagesSeed from 'src/core/pipeline/seed-data/pipeline-stages.json';
+import pipelinesSeed from 'src/core/pipeline/seed-data/sales-pipeline.json';
+import { arraysEqual } from 'src/utils/equal';
+import { WorkspaceService } from 'src/core/workspace/services/workspace.service';
+
+interface DataCleanInactiveOptions {
+  days?: number;
+  sameAsSeedDays?: number;
+  dryRun?: boolean;
+  confirmation?: boolean;
+}
+
+interface ActivityReport {
+  displayName: string;
+  maxUpdatedAt: string;
+  inactiveDays: number;
+}
+
+interface SameAsSeedWorkspace {
+  displayName: string;
+}
+
+interface DataCleanResults {
+  activityReport: { [key: string]: ActivityReport };
+  sameAsSeedWorkspaces: { [key: string]: SameAsSeedWorkspace };
+}
+
+@Command({
+  name: 'workspaces:clean-inactive',
+  description: 'Clean inactive workspaces from the public database schema',
+})
+export class DataCleanInactiveCommand extends CommandRunner {
+  constructor(
+    private readonly prismaService: PrismaService,
+    private readonly workspaceService: WorkspaceService,
+    private readonly inquiererService: InquirerService,
+  ) {
+    super();
+  }
+
+  @Option({
+    flags: '-d, --days [inactive days threshold]',
+    description: 'Inactive days threshold',
+    defaultValue: 60,
+  })
+  parseDays(val: string): number {
+    return Number(val);
+  }
+
+  @Option({
+    flags: '-s, --same-as-seed-days [same as seed days threshold]',
+    description: 'Same as seed days threshold',
+    defaultValue: 10,
+  })
+  parseSameAsSeedDays(val: string): number {
+    return Number(val);
+  }
+
+  @Option({
+    flags: '--dry-run [dry run]',
+    description: 'List inactive workspaces without removing them',
+  })
+  parseDryRun(val: string): boolean {
+    return Boolean(val);
+  }
+
+  // We look for public tables which contains workspaceId and updatedAt columns
+  getRelevantTables() {
+    return Object.keys(this.prismaService.client).filter(
+      (name) =>
+        !name.startsWith('_') &&
+        !name.startsWith('$') &&
+        !name.includes('user') &&
+        !name.includes('refreshToken') &&
+        !name.includes('workspace'),
+    );
+  }
+
+  async getTableMaxUpdatedAt(table, workspace) {
+    try {
+      return await this.prismaService.client[table].aggregate({
+        _max: { updatedAt: true },
+        where: { workspaceId: { equals: workspace.id } },
+      });
+    } catch (e) {}
+  }
+
+  updateResult(result, workspace, newUpdatedAt) {
+    if (!result.activityReport[workspace.id]) {
+      result.activityReport[workspace.id] = {
+        displayName: workspace.displayName,
+        maxUpdatedAt: null,
+      };
+    }
+    if (
+      newUpdatedAt &&
+      newUpdatedAt._max.updatedAt &&
+      new Date(result.activityReport[workspace.id].maxUpdatedAt) <
+        new Date(newUpdatedAt._max.updatedAt)
+    ) {
+      result.activityReport[workspace.id].maxUpdatedAt =
+        newUpdatedAt._max.updatedAt;
+    }
+  }
+
+  async detectWorkspacesWithSeedDataOnly(result, workspace) {
+    const companies = await this.prismaService.client.company.findMany({
+      select: { name: true, domainName: true, address: true, employees: true },
+      where: { workspaceId: { equals: workspace.id } },
+    });
+    const people = await this.prismaService.client.person.findMany({
+      select: {
+        firstName: true,
+        lastName: true,
+        city: true,
+        email: true,
+        avatarUrl: true,
+      },
+      where: { workspaceId: { equals: workspace.id } },
+    });
+    const pipelineStages =
+      await this.prismaService.client.pipelineStage.findMany({
+        select: {
+          name: true,
+          color: true,
+          position: true,
+          type: true,
+        },
+        where: { workspaceId: { equals: workspace.id } },
+      });
+    const pipelines = await this.prismaService.client.pipeline.findMany({
+      select: {
+        name: true,
+        icon: true,
+        pipelineProgressableType: true,
+      },
+      where: { workspaceId: { equals: workspace.id } },
+    });
+    if (
+      arraysEqual(people, peopleSeed) &&
+      arraysEqual(companies, companiesSeed) &&
+      arraysEqual(pipelineStages, pipelineStagesSeed) &&
+      arraysEqual(pipelines, [pipelinesSeed])
+    ) {
+      result.sameAsSeedWorkspaces[workspace.id] = {
+        displayName: workspace.displayName,
+      };
+    }
+  }
+
+  async findInactiveWorkspaces(result) {
+    const workspaces = await this.prismaService.client.workspace.findMany();
+    const tables = this.getRelevantTables();
+    for (const workspace of workspaces) {
+      await this.detectWorkspacesWithSeedDataOnly(result, workspace);
+      for (const table of tables) {
+        const maxUpdatedAt = await this.getTableMaxUpdatedAt(table, workspace);
+        this.updateResult(result, workspace, maxUpdatedAt);
+      }
+    }
+  }
+
+  filterResults(result, options) {
+    for (const workspaceId in result.activityReport) {
+      const timeDifferenceInSeconds = Math.abs(
+        new Date().getTime() -
+          new Date(result.activityReport[workspaceId].maxUpdatedAt).getTime(),
+      );
+      const timeDifferenceInDays = Math.ceil(
+        timeDifferenceInSeconds / (1000 * 3600 * 24),
+      );
+      if (timeDifferenceInDays < options.sameAsSeedDays) {
+        delete result.sameAsSeedWorkspaces[workspaceId];
+      }
+      if (timeDifferenceInDays < options.days) {
+        delete result.activityReport[workspaceId];
+      } else {
+        result.activityReport[workspaceId].inactiveDays = timeDifferenceInDays;
+      }
+    }
+  }
+
+  async delete(result) {
+    if (Object.keys(result.activityReport).length) {
+      console.log('Deleting inactive workspaces');
+    }
+    for (const workspaceId in result.activityReport) {
+      await this.workspaceService.deleteWorkspace({
+        workspaceId,
+      });
+      console.log(`- ${workspaceId} deleted`);
+    }
+    if (Object.keys(result.sameAsSeedWorkspaces).length) {
+      console.log('Deleting same as Seed workspaces');
+    }
+    for (const workspaceId in result.sameAsSeedWorkspaces) {
+      await this.workspaceService.deleteWorkspace({
+        workspaceId,
+      });
+      console.log(`- ${workspaceId} deleted`);
+    }
+  }
+
+  async run(
+    _passedParam: string[],
+    options: DataCleanInactiveOptions,
+  ): Promise<void> {
+    if (!options.dryRun) {
+      options = await this.inquiererService.ask('confirm', options);
+      if (!options.confirmation) {
+        console.log('Cleaning aborted');
+        return;
+      }
+    }
+    const result: DataCleanResults = {
+      activityReport: {},
+      sameAsSeedWorkspaces: {},
+    };
+    await this.findInactiveWorkspaces(result);
+    this.filterResults(result, options);
+    if (!options.dryRun) {
+      await this.delete(result);
+    } else {
+      console.log(result);
+    }
+  }
+}

server/src/database/commands/confirmation.question.ts

@@ -0,0 +1,16 @@
+import { Question, QuestionSet } from 'nest-commander';
+
+@QuestionSet({
+  name: 'confirm',
+})
+export class ConfirmationQuestion {
+  @Question({
+    type: 'confirm',
+    name: 'confirmation',
+    message:
+      "You are about to delete data from database. Are you sure to continue? Consider the '--dry-run' option first",
+  })
+  parseConfirm(val: string): boolean {
+    return Boolean(val);
+  }
+}

server/src/database/commands/database-command.module.ts

@@ -0,0 +1,22 @@
+import { Module } from '@nestjs/common';
+
+import { DataCleanInactiveCommand } from 'src/database/commands/clean-inactive-workspaces.command';
+import { ConfirmationQuestion } from 'src/database/commands/confirmation.question';
+import { WorkspaceService } from 'src/core/workspace/services/workspace.service';
+import { PipelineModule } from 'src/core/pipeline/pipeline.module';
+import { CompanyModule } from 'src/core/company/company.module';
+import { PersonModule } from 'src/core/person/person.module';
+import { TenantInitialisationModule } from 'src/metadata/tenant-initialisation/tenant-initialisation.module';
+import { PrismaModule } from 'src/database/prisma.module';
+
+@Module({
+  imports: [
+    PipelineModule,
+    CompanyModule,
+    PersonModule,
+    TenantInitialisationModule,
+    PrismaModule,
+  ],
+  providers: [DataCleanInactiveCommand, ConfirmationQuestion, WorkspaceService],
+})
+export class DatabaseCommandModule {}

server/src/utils/equal.ts

@@ -0,0 +1,13 @@
+//https://stackoverflow.com/questions/27030/comparing-arrays-of-objects-in-javascript
+export const objectsEqual = (o1, o2) => {
+  return (
+    Object.keys(o1).length === Object.keys(o2).length &&
+    Object.keys(o1).every((p) => o1[p] === o2[p])
+  );
+};
+
+export const arraysEqual = (a1, a2) => {
+  return (
+    a1.length === a2.length && a1.every((o, idx) => objectsEqual(o, a2[idx]))
+  );
+};