6181 workflows create a custom code executor (#6235)

Closes #6181

## Testing
- download Altair graphql dev tool https://altairgraphql.dev/#download
- create a file locally `test.ts` containing:
```
export const handler = async (event: object, context: object) => {
  return { test: 'toto', data: event['data'] };
}
```
- play those requests in Altair:
mutation UpsertFunction($file: Upload!) {
  upsertFunction(name: "toto", file: $file)
}

mutation ExecFunction {
  executeFunction(name:"toto", payload: {data: "titi"})
}
- it will run the local driver, add those env variable to test with
lambda driver
```
CUSTOM_CODE_ENGINE_DRIVER_TYPE=lambda
LAMBDA_REGION=eu-west-2
LAMBDA_ROLE=<ASK_ME>
```
This commit is contained in:
martmull
2024-07-17 17:53:01 +02:00
committed by GitHub
parent e6f6069fe7
commit 47ddc7be83
37 changed files with 2382 additions and 16 deletions

View File

@@ -3,6 +3,7 @@
"@air/react-drag-to-select": "^5.0.8",
"@apollo/client": "^3.7.17",
"@apollo/server": "^4.7.3",
"@aws-sdk/client-lambda": "^3.614.0",
"@aws-sdk/client-s3": "^3.363.0",
"@aws-sdk/credential-providers": "^3.363.0",
"@blocknote/core": "^0.12.1",
@@ -70,6 +71,8 @@
"afterframe": "^1.0.2",
"apollo-server-express": "^3.12.0",
"apollo-upload-client": "^17.0.0",
"archiver": "^7.0.1",
"aws-sdk": "^2.1658.0",
"axios": "^1.6.2",
"bcrypt": "^5.1.1",
"better-sqlite3": "^9.2.2",
@@ -93,6 +96,7 @@
"facepaint": "^1.2.1",
"file-type": "16.5.4",
"framer-motion": "^10.12.17",
"fs-extra": "^11.2.0",
"googleapis": "105",
"graphiql": "^3.1.1",
"graphql": "16.8.0",

View File

@@ -26,6 +26,13 @@ export type Scalars = {
Upload: { input: any; output: any; }
};
export type AisqlQueryResult = {
__typename?: 'AISQLQueryResult';
queryFailedErrorMessage?: Maybe<Scalars['String']['output']>;
sqlQuery: Scalars['String']['output'];
sqlQueryResult?: Maybe<Scalars['String']['output']>;
};
export type ActivateWorkspaceInput = {
displayName?: InputMaybe<Scalars['String']['input']>;
};
@@ -345,6 +352,7 @@ export enum FileFolder {
Attachment = 'Attachment',
PersonPicture = 'PersonPicture',
ProfilePicture = 'ProfilePicture',
ServerlessFunction = 'ServerlessFunction',
WorkspaceLogo = 'WorkspaceLogo'
}
@@ -397,6 +405,7 @@ export type Mutation = {
createOneObject: Object;
createOneRelation: Relation;
createOneRemoteServer: RemoteServer;
createOneServerlessFunction: ServerlessFunction;
deleteCurrentWorkspace: Workspace;
deleteOneField: Field;
deleteOneObject: Object;
@@ -407,6 +416,7 @@ export type Mutation = {
emailPasswordResetLink: EmailPasswordResetLink;
enablePostgresProxy: PostgresCredentials;
exchangeAuthorizationCode: ExchangeAuthCode;
executeOneServerlessFunction: ServerlessFunctionExecutionResult;
generateApiKeyToken: ApiKeyToken;
generateJWT: AuthTokens;
generateTransientToken: TransientToken;
@@ -488,6 +498,12 @@ export type MutationCreateOneRemoteServerArgs = {
};
export type MutationCreateOneServerlessFunctionArgs = {
file: Scalars['Upload']['input'];
name: Scalars['String']['input'];
};
export type MutationDeleteOneFieldArgs = {
input: DeleteOneFieldInput;
};
@@ -520,6 +536,12 @@ export type MutationExchangeAuthorizationCodeArgs = {
};
export type MutationExecuteOneServerlessFunctionArgs = {
name: Scalars['String']['input'];
payload?: InputMaybe<Scalars['JSON']['input']>;
};
export type MutationGenerateApiKeyTokenArgs = {
apiKeyId: Scalars['String']['input'];
expiresAt: Scalars['String']['input'];
@@ -707,6 +729,7 @@ export type Query = {
findManyRemoteServersByType: Array<RemoteServer>;
findOneRemoteServerById: RemoteServer;
findWorkspaceFromInviteHash: Workspace;
getAISQLQuery: AisqlQueryResult;
getPostgresCredentials?: Maybe<PostgresCredentials>;
getProductPrices: ProductPricesEntity;
getTimelineCalendarEventsFromCompanyId: TimelineCalendarEventsWithTotal;
@@ -717,6 +740,8 @@ export type Query = {
objects: ObjectConnection;
relation: Relation;
relations: RelationConnection;
serverlessFunction: ServerlessFunction;
serverlessFunctions: ServerlessFunctionConnection;
validatePasswordResetToken: ValidatePasswordResetToken;
};
@@ -768,6 +793,11 @@ export type QueryFindWorkspaceFromInviteHashArgs = {
};
export type QueryGetAisqlQueryArgs = {
text: Scalars['String']['input'];
};
export type QueryGetProductPricesArgs = {
product: Scalars['String']['input'];
};
@@ -822,6 +852,18 @@ export type QueryRelationsArgs = {
};
export type QueryServerlessFunctionArgs = {
id: Scalars['UUID']['input'];
};
export type QueryServerlessFunctionsArgs = {
filter?: ServerlessFunctionFilter;
paging?: CursorPaging;
sorting?: Array<ServerlessFunctionSort>;
};
export type QueryValidatePasswordResetTokenArgs = {
passwordResetToken: Scalars['String']['input'];
};
@@ -915,6 +957,26 @@ export type Sentry = {
release?: Maybe<Scalars['String']['output']>;
};
export type ServerlessFunctionConnection = {
__typename?: 'ServerlessFunctionConnection';
/** Array of edges. */
edges: Array<ServerlessFunctionEdge>;
/** Paging information */
pageInfo: PageInfo;
};
export type ServerlessFunctionExecutionResult = {
__typename?: 'ServerlessFunctionExecutionResult';
/** Execution result in JSON format */
result: Scalars['JSON']['output'];
};
/** SyncStatus of the serverlessFunction */
export enum ServerlessFunctionSyncStatus {
NotReady = 'NOT_READY',
Ready = 'READY'
}
export type SessionEntity = {
__typename?: 'SessionEntity';
url?: Maybe<Scalars['String']['output']>;
@@ -1345,6 +1407,39 @@ export type RelationEdge = {
node: Relation;
};
export type ServerlessFunction = {
__typename?: 'serverlessFunction';
createdAt: Scalars['DateTime']['output'];
id: Scalars['UUID']['output'];
name: Scalars['String']['output'];
syncStatus: ServerlessFunctionSyncStatus;
updatedAt: Scalars['DateTime']['output'];
};
export type ServerlessFunctionEdge = {
__typename?: 'serverlessFunctionEdge';
/** Cursor for this node. */
cursor: Scalars['ConnectionCursor']['output'];
/** The node containing the serverlessFunction */
node: ServerlessFunction;
};
export type ServerlessFunctionFilter = {
and?: InputMaybe<Array<ServerlessFunctionFilter>>;
id?: InputMaybe<UuidFilterComparison>;
or?: InputMaybe<Array<ServerlessFunctionFilter>>;
};
export type ServerlessFunctionSort = {
direction: SortDirection;
field: ServerlessFunctionSortFields;
nulls?: InputMaybe<SortNulls>;
};
export enum ServerlessFunctionSortFields {
Id = 'id'
}
export type RemoteServerFieldsFragment = { __typename?: 'RemoteServer', id: string, createdAt: any, foreignDataWrapperId: string, foreignDataWrapperOptions?: any | null, foreignDataWrapperType: string, updatedAt: any, schema?: string | null, label: string, userMappingOptions?: { __typename?: 'UserMappingOptionsUser', user?: string | null } | null };
export type RemoteTableFieldsFragment = { __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<DistantTableUpdate> | null };

View File

@@ -258,6 +258,7 @@ export enum FileFolder {
Attachment = 'Attachment',
PersonPicture = 'PersonPicture',
ProfilePicture = 'ProfilePicture',
ServerlessFunction = 'ServerlessFunction',
WorkspaceLogo = 'WorkspaceLogo'
}
@@ -683,6 +684,26 @@ export type Sentry = {
release?: Maybe<Scalars['String']>;
};
export type ServerlessFunctionConnection = {
__typename?: 'ServerlessFunctionConnection';
/** Array of edges. */
edges: Array<ServerlessFunctionEdge>;
/** Paging information */
pageInfo: PageInfo;
};
export type ServerlessFunctionExecutionResult = {
__typename?: 'ServerlessFunctionExecutionResult';
/** Execution result in JSON format */
result: Scalars['JSON'];
};
/** SyncStatus of the serverlessFunction */
export enum ServerlessFunctionSyncStatus {
NotReady = 'NOT_READY',
Ready = 'READY'
}
export type SessionEntity = {
__typename?: 'SessionEntity';
url?: Maybe<Scalars['String']>;
@@ -1064,6 +1085,23 @@ export type RelationEdge = {
node: Relation;
};
export type ServerlessFunction = {
__typename?: 'serverlessFunction';
createdAt: Scalars['DateTime'];
id: Scalars['UUID'];
name: Scalars['String'];
syncStatus: ServerlessFunctionSyncStatus;
updatedAt: Scalars['DateTime'];
};
export type ServerlessFunctionEdge = {
__typename?: 'serverlessFunctionEdge';
/** Cursor for this node. */
cursor: Scalars['ConnectionCursor'];
/** The node containing the serverlessFunction */
node: ServerlessFunction;
};
export type TimelineCalendarEventFragmentFragment = { __typename?: 'TimelineCalendarEvent', id: any, title: string, description: string, location: string, startsAt: string, endsAt: string, isFullDay: boolean, visibility: CalendarChannelVisibility, participants: Array<{ __typename?: 'TimelineCalendarEventParticipant', personId?: any | null, workspaceMemberId?: any | null, firstName: string, lastName: string, displayName: string, avatarUrl: string, handle: string }> };
export type TimelineCalendarEventParticipantFragmentFragment = { __typename?: 'TimelineCalendarEventParticipant', personId?: any | null, workspaceMemberId?: any | null, firstName: string, lastName: string, displayName: string, avatarUrl: string, handle: string };

View File

@@ -35,6 +35,7 @@ SIGN_IN_PREFILLED=true
# AUTH_GOOGLE_CLIENT_SECRET=replace_me_with_google_client_secret
# AUTH_GOOGLE_CALLBACK_URL=http://localhost:3000/auth/google/redirect
# AUTH_GOOGLE_APIS_CALLBACK_URL=http://localhost:3000/auth/google-apis/get-access-token
# SERVERLESS_TYPE=local
# STORAGE_TYPE=local
# STORAGE_LOCAL_PATH=.local-storage
# SUPPORT_DRIVER=front

View File

@@ -0,0 +1,23 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class CreateServerlessFunctionTable1721210534680
implements MigrationInterface
{
name = 'CreateServerlessFunctionTable1721210534680';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TYPE "metadata"."serverlessFunction_syncstatus_enum" AS ENUM('NOT_READY', 'READY')`,
);
await queryRunner.query(
`CREATE TABLE "metadata"."serverlessFunction" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "sourceCodeHash" character varying NOT NULL, "syncStatus" "metadata"."serverlessFunction_syncstatus_enum" NOT NULL DEFAULT 'NOT_READY', "workspaceId" uuid NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "IndexOnNameAndWorkspaceIdUnique" UNIQUE ("name", "workspaceId"), CONSTRAINT "PK_49bfacee064bee9d0d486483b60" PRIMARY KEY ("id"))`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "metadata"."serverlessFunction"`);
await queryRunner.query(
`DROP TYPE "metadata"."serverlessFunction_syncstatus_enum"`,
);
}
}

View File

@@ -13,9 +13,9 @@ import { HealthModule } from 'src/engine/core-modules/health/health.module';
import { AISQLQueryModule } from 'src/engine/core-modules/ai-sql-query/ai-sql-query.module';
import { PostgresCredentialsModule } from 'src/engine/core-modules/postgres-credentials/postgres-credentials.module';
import { AnalyticsModule } from './analytics/analytics.module';
import { FileModule } from './file/file.module';
import { ClientConfigModule } from './client-config/client-config.module';
import { FileModule } from './file/file.module';
import { AnalyticsModule } from './analytics/analytics.module';
@Module({
imports: [

View File

@@ -5,6 +5,7 @@ export enum FileFolder {
WorkspaceLogo = 'workspace-logo',
Attachment = 'attachment',
PersonPicture = 'person-picture',
ServerlessFunction = 'serverless-function',
}
registerEnumType(FileFolder, {

View File

@@ -20,6 +20,7 @@ import { NodeEnvironment } from 'src/engine/integrations/environment/interfaces/
import { LLMChatModelDriver } from 'src/engine/integrations/llm-chat-model/interfaces/llm-chat-model.interface';
import { LLMTracingDriver } from 'src/engine/integrations/llm-tracing/interfaces/llm-tracing.interface';
import { ServerlessDriverType } from 'src/engine/integrations/serverless/serverless.interface';
import { assert } from 'src/utils/assert';
import { CastToStringArray } from 'src/engine/integrations/environment/decorators/cast-to-string-array.decorator';
import { ExceptionHandlerDriver } from 'src/engine/integrations/exception-handler/interfaces';
@@ -204,6 +205,30 @@ export class EnvironmentVariables {
@ValidateIf((env) => env.AUTH_GOOGLE_ENABLED)
AUTH_GOOGLE_CALLBACK_URL: string;
// Custom Code Engine
@IsEnum(ServerlessDriverType)
@IsOptional()
SERVERLESS_TYPE: ServerlessDriverType = ServerlessDriverType.Local;
@ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda)
@IsAWSRegion()
SERVERLESS_LAMBDA_REGION: AwsRegion;
@ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda)
@IsString()
@IsOptional()
SERVERLESS_LAMBDA_ROLE: string;
@ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda)
@IsString()
@IsOptional()
SERVERLESS_LAMBDA_ACCESS_KEY_ID: string;
@ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda)
@IsString()
@IsOptional()
SERVERLESS_LAMBDA_SECRET_ACCESS_KEY: string;
// Storage
@IsEnum(StorageDriverType)
@IsOptional()

View File

@@ -0,0 +1,11 @@
import { Readable } from 'stream';
export const readFileContent = async (stream: Readable): Promise<string> => {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.from(chunk));
}
return Buffer.concat(chunks).toString('utf8');
};

View File

@@ -16,6 +16,10 @@ import { LLMChatModelModule } from 'src/engine/integrations/llm-chat-model/llm-c
import { llmChatModelModuleFactory } from 'src/engine/integrations/llm-chat-model/llm-chat-model.module-factory';
import { LLMTracingModule } from 'src/engine/integrations/llm-tracing/llm-tracing.module';
import { llmTracingModuleFactory } from 'src/engine/integrations/llm-tracing/llm-tracing.module-factory';
import { ServerlessModule } from 'src/engine/integrations/serverless/serverless.module';
import { serverlessModuleFactory } from 'src/engine/integrations/serverless/serverless-module.factory';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
import { EnvironmentModule } from './environment/environment.module';
import { EnvironmentService } from './environment/environment.service';
@@ -62,6 +66,14 @@ import { MessageQueueModule } from './message-queue/message-queue.module';
useFactory: llmTracingModuleFactory,
inject: [EnvironmentService],
}),
ServerlessModule.forRootAsync({
useFactory: serverlessModuleFactory,
inject: [
EnvironmentService,
FileStorageService,
BuildDirectoryManagerService,
],
}),
],
exports: [],
providers: [],

View File

@@ -0,0 +1,33 @@
import { join } from 'path';
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { SOURCE_FILE_NAME } from 'src/engine/integrations/serverless/drivers/constants/source-file-name';
import { readFileContent } from 'src/engine/integrations/file-storage/utils/read-file-content';
import { compileTypescript } from 'src/engine/integrations/serverless/drivers/utils/compile-typescript';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
export class BaseServerlessDriver {
getFolderPath(serverlessFunction: ServerlessFunctionEntity) {
return join(
FileFolder.ServerlessFunction,
serverlessFunction.workspaceId,
serverlessFunction.id,
);
}
async getCompiledCode(
serverlessFunction: ServerlessFunctionEntity,
fileStorageService: FileStorageService,
) {
const folderPath = this.getFolderPath(serverlessFunction);
const fileStream = await fileStorageService.read({
folderPath,
filename: SOURCE_FILE_NAME,
});
const typescriptCode = await readFileContent(fileStream);
return compileTypescript(typescriptCode);
}
}

View File

@@ -0,0 +1 @@
export const BUILD_FILE_NAME = 'build.js';

View File

@@ -0,0 +1 @@
export const SOURCE_FILE_NAME = 'source.ts';

View File

@@ -0,0 +1,9 @@
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
export interface ServerlessDriver {
build(serverlessFunction: ServerlessFunctionEntity): Promise<void>;
execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object | undefined,
): Promise<object>;
}

View File

@@ -0,0 +1,100 @@
import fs from 'fs';
import {
CreateFunctionCommand,
Lambda,
LambdaClientConfig,
InvokeCommand,
} from '@aws-sdk/client-lambda';
import { CreateFunctionCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/CreateFunctionCommand';
import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import { createZipFile } from 'src/engine/integrations/serverless/drivers/utils/create-zip-file';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
export interface LambdaDriverOptions extends LambdaClientConfig {
fileStorageService: FileStorageService;
buildDirectoryManagerService: BuildDirectoryManagerService;
region: string;
role: string;
}
export class LambdaDriver
extends BaseServerlessDriver
implements ServerlessDriver
{
private readonly lambdaClient: Lambda;
private readonly lambdaRole: string;
private readonly fileStorageService: FileStorageService;
private readonly buildDirectoryManagerService: BuildDirectoryManagerService;
constructor(options: LambdaDriverOptions) {
super();
const { region, role, ...lambdaOptions } = options;
this.lambdaClient = new Lambda({ ...lambdaOptions, region });
this.lambdaRole = role;
this.fileStorageService = options.fileStorageService;
this.buildDirectoryManagerService = options.buildDirectoryManagerService;
}
async build(serverlessFunction: ServerlessFunctionEntity) {
const javascriptCode = await this.getCompiledCode(
serverlessFunction,
this.fileStorageService,
);
const {
sourceTemporaryDir,
lambdaZipPath,
javascriptFilePath,
lambdaHandler,
} = await this.buildDirectoryManagerService.init();
await fs.promises.writeFile(javascriptFilePath, javascriptCode);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const params: CreateFunctionCommandInput = {
Code: {
ZipFile: await fs.promises.readFile(lambdaZipPath),
},
FunctionName: serverlessFunction.id,
Handler: lambdaHandler,
Role: this.lambdaRole,
Runtime: 'nodejs18.x',
Description: 'Lambda function to run user script',
Timeout: 900,
};
const command = new CreateFunctionCommand(params);
await this.lambdaClient.send(command);
await this.buildDirectoryManagerService.clean();
}
async execute(
functionToExecute: ServerlessFunctionEntity,
payload: object | undefined = undefined,
): Promise<object> {
const params = {
FunctionName: functionToExecute.id,
Payload: JSON.stringify(payload),
};
const command = new InvokeCommand(params);
const result = await this.lambdaClient.send(command);
if (!result.Payload) {
return {};
}
return JSON.parse(result.Payload.transformToString());
}
}

View File

@@ -0,0 +1,94 @@
import { join } from 'path';
import { tmpdir } from 'os';
import { promises as fs } from 'fs';
import { fork } from 'child_process';
import { v4 } from 'uuid';
import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { readFileContent } from 'src/engine/integrations/file-storage/utils/read-file-content';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { BUILD_FILE_NAME } from 'src/engine/integrations/serverless/drivers/constants/build-file-name';
import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
export interface LocalDriverOptions {
fileStorageService: FileStorageService;
}
export class LocalDriver
extends BaseServerlessDriver
implements ServerlessDriver
{
private readonly fileStorageService: FileStorageService;
constructor(options: LocalDriverOptions) {
super();
this.fileStorageService = options.fileStorageService;
}
async build(serverlessFunction: ServerlessFunctionEntity) {
const javascriptCode = await this.getCompiledCode(
serverlessFunction,
this.fileStorageService,
);
await this.fileStorageService.write({
file: javascriptCode,
name: BUILD_FILE_NAME,
mimeType: undefined,
folder: this.getFolderPath(serverlessFunction),
});
}
async execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object | undefined = undefined,
): Promise<object> {
const fileStream = await this.fileStorageService.read({
folderPath: this.getFolderPath(serverlessFunction),
filename: BUILD_FILE_NAME,
});
const fileContent = await readFileContent(fileStream);
const tmpFilePath = join(tmpdir(), `${v4()}.js`);
const modifiedContent = `
process.on('message', async (message) => {
const { event, context } = message;
const result = await handler(event, context);
process.send(result);
});
${fileContent}
`;
await fs.writeFile(tmpFilePath, modifiedContent);
return await new Promise((resolve, reject) => {
const child = fork(tmpFilePath);
child.on('message', (message: object) => {
resolve(message);
child.kill();
fs.unlink(tmpFilePath);
});
child.on('error', (error) => {
reject(error);
child.kill();
fs.unlink(tmpFilePath);
});
child.on('exit', (code) => {
if (code && code !== 0) {
reject(new Error(`Child process exited with code ${code}`));
fs.unlink(tmpFilePath);
}
});
child.send({ event: payload });
});
}
}

View File

@@ -0,0 +1,48 @@
import { Injectable } from '@nestjs/common';
import { join } from 'path';
import { tmpdir } from 'os';
import fs from 'fs';
import fsExtra from 'fs-extra';
import { v4 } from 'uuid';
const TEMPORARY_LAMBDA_FOLDER = 'twenty-build-lambda-temp-folder';
const TEMPORARY_LAMBDA_SOURCE_FOLDER = 'src';
const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
const LAMBDA_ENTRY_FILE_NAME = 'index.js';
@Injectable()
export class BuildDirectoryManagerService {
private temporaryDir = join(tmpdir(), `${TEMPORARY_LAMBDA_FOLDER}_${v4()}`);
private lambdaHandler = `${LAMBDA_ENTRY_FILE_NAME.split('.')[0]}.handler`;
async init() {
const sourceTemporaryDir = join(
this.temporaryDir,
TEMPORARY_LAMBDA_SOURCE_FOLDER,
);
const lambdaZipPath = join(this.temporaryDir, LAMBDA_ZIP_FILE_NAME);
const javascriptFilePath = join(sourceTemporaryDir, LAMBDA_ENTRY_FILE_NAME);
if (!fs.existsSync(this.temporaryDir)) {
await fs.promises.mkdir(this.temporaryDir);
await fs.promises.mkdir(sourceTemporaryDir);
} else {
await fsExtra.emptyDir(this.temporaryDir);
await fs.promises.mkdir(sourceTemporaryDir);
}
return {
sourceTemporaryDir,
lambdaZipPath,
javascriptFilePath,
lambdaHandler: this.lambdaHandler,
};
}
async clean() {
await fsExtra.emptyDir(this.temporaryDir);
await fs.promises.rmdir(this.temporaryDir);
}
}

View File

@@ -0,0 +1,19 @@
import ts from 'typescript';
export const compileTypescript = (typescriptCode: string): string => {
const options: ts.CompilerOptions = {
module: ts.ModuleKind.CommonJS,
target: ts.ScriptTarget.ES2017,
moduleResolution: ts.ModuleResolutionKind.Node10,
esModuleInterop: true,
resolveJsonModule: true,
allowSyntheticDefaultImports: true,
types: ['node'],
};
const result = ts.transpileModule(typescriptCode, {
compilerOptions: options,
});
return result.outputText;
};

View File

@@ -0,0 +1,21 @@
import fs from 'fs';
import { pipeline } from 'stream/promises';
import archiver from 'archiver';
export const createZipFile = async (
sourceDir: string,
outPath: string,
): Promise<void> => {
const output = fs.createWriteStream(outPath);
const archive = archiver('zip', {
zlib: { level: 9 }, // Compression level
});
const p = pipeline(archive, output);
archive.directory(sourceDir, false);
archive.finalize();
return p;
};

View File

@@ -0,0 +1,59 @@
import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
import {
ServerlessModuleOptions,
ServerlessDriverType,
} from 'src/engine/integrations/serverless/serverless.interface';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
export const serverlessModuleFactory = async (
environmentService: EnvironmentService,
fileStorageService: FileStorageService,
buildDirectoryManagerService: BuildDirectoryManagerService,
): Promise<ServerlessModuleOptions> => {
const driverType = environmentService.get('SERVERLESS_TYPE');
const options = { fileStorageService };
switch (driverType) {
case ServerlessDriverType.Local: {
return {
type: ServerlessDriverType.Local,
options,
};
}
case ServerlessDriverType.Lambda: {
const region = environmentService.get('SERVERLESS_LAMBDA_REGION');
const accessKeyId = environmentService.get(
'SERVERLESS_LAMBDA_ACCESS_KEY_ID',
);
const secretAccessKey = environmentService.get(
'SERVERLESS_LAMBDA_SECRET_ACCESS_KEY',
);
const role = environmentService.get('SERVERLESS_LAMBDA_ROLE');
return {
type: ServerlessDriverType.Lambda,
options: {
...options,
buildDirectoryManagerService,
credentials: accessKeyId
? {
accessKeyId,
secretAccessKey,
}
: fromNodeProviderChain({
clientConfig: { region },
}),
region: region ?? '',
role: role ?? '',
},
};
}
default:
throw new Error(
`Invalid serverless driver type (${driverType}), check your .env file`,
);
}
};

View File

@@ -0,0 +1 @@
export const SERVERLESS_DRIVER = Symbol('SERVERLESS_DRIVER');

View File

@@ -0,0 +1,30 @@
import { FactoryProvider, ModuleMetadata } from '@nestjs/common';
import { LocalDriverOptions } from 'src/engine/integrations/serverless/drivers/local.driver';
import { LambdaDriverOptions } from 'src/engine/integrations/serverless/drivers/lambda.driver';
export enum ServerlessDriverType {
Lambda = 'lambda',
Local = 'local',
}
export interface LocalDriverFactoryOptions {
type: ServerlessDriverType.Local;
options: LocalDriverOptions;
}
export interface LambdaDriverFactoryOptions {
type: ServerlessDriverType.Lambda;
options: LambdaDriverOptions;
}
export type ServerlessModuleOptions =
| LocalDriverFactoryOptions
| LambdaDriverFactoryOptions;
export type ServerlessModuleAsyncOptions = {
useFactory: (
...args: any[]
) => ServerlessModuleOptions | Promise<ServerlessModuleOptions>;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;

View File

@@ -0,0 +1,35 @@
import { DynamicModule, Global } from '@nestjs/common';
import {
ServerlessDriverType,
ServerlessModuleAsyncOptions,
} from 'src/engine/integrations/serverless/serverless.interface';
import { ServerlessService } from 'src/engine/integrations/serverless/serverless.service';
import { SERVERLESS_DRIVER } from 'src/engine/integrations/serverless/serverless.constants';
import { LocalDriver } from 'src/engine/integrations/serverless/drivers/local.driver';
import { LambdaDriver } from 'src/engine/integrations/serverless/drivers/lambda.driver';
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
@Global()
export class ServerlessModule {
static forRootAsync(options: ServerlessModuleAsyncOptions): DynamicModule {
const provider = {
provide: SERVERLESS_DRIVER,
useFactory: async (...args: any[]) => {
const config = await options.useFactory(...args);
return config?.type === ServerlessDriverType.Local
? new LocalDriver(config.options)
: new LambdaDriver(config.options);
},
inject: options.inject || [],
};
return {
module: ServerlessModule,
imports: options.imports || [],
providers: [ServerlessService, BuildDirectoryManagerService, provider],
exports: [ServerlessService],
};
}
}

View File

@@ -0,0 +1,22 @@
import { Inject, Injectable } from '@nestjs/common';
import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import { SERVERLESS_DRIVER } from 'src/engine/integrations/serverless/serverless.constants';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
@Injectable()
export class ServerlessService implements ServerlessDriver {
constructor(@Inject(SERVERLESS_DRIVER) private driver: ServerlessDriver) {}
async build(serverlessFunction: ServerlessFunctionEntity): Promise<void> {
return this.driver.build(serverlessFunction);
}
async execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object | undefined = undefined,
) {
return this.driver.execute(serverlessFunction, payload);
}
}

View File

@@ -7,6 +7,7 @@ import { RelationMetadataModule } from 'src/engine/metadata-modules/relation-met
import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module';
@Module({
imports: [
@@ -14,6 +15,7 @@ import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-
FieldMetadataModule,
ObjectMetadataModule,
RelationMetadataModule,
ServerlessFunctionModule,
WorkspaceCacheVersionModule,
WorkspaceMigrationModule,
RemoteServerModule,
@@ -24,6 +26,7 @@ import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-
FieldMetadataModule,
ObjectMetadataModule,
RelationMetadataModule,
ServerlessFunctionModule,
RemoteServerModule,
],
})

View File

@@ -0,0 +1,20 @@
import { ArgsType, Field } from '@nestjs/graphql';
import { IsNotEmpty, IsObject, IsOptional, IsString } from 'class-validator';
import graphqlTypeJson from 'graphql-type-json';
@ArgsType()
export class ExecuteServerlessFunctionInput {
@Field({ description: 'Name of the serverless function to execute' })
@IsNotEmpty()
@IsString()
name: string;
@Field(() => graphqlTypeJson, {
description: 'Payload in JSON format',
nullable: true,
})
@IsObject()
@IsOptional()
payload?: JSON;
}

View File

@@ -0,0 +1,13 @@
import { Field, ObjectType } from '@nestjs/graphql';
import { IsObject } from 'class-validator';
import graphqlTypeJson from 'graphql-type-json';
@ObjectType('ServerlessFunctionExecutionResult')
export class ServerlessFunctionExecutionResultDTO {
@IsObject()
@Field(() => graphqlTypeJson, {
description: 'Execution result in JSON format',
})
result: JSON;
}

View File

@@ -0,0 +1,70 @@
import {
Field,
HideField,
ObjectType,
registerEnumType,
} from '@nestjs/graphql';
import {
Authorize,
IDField,
QueryOptions,
} from '@ptc-org/nestjs-query-graphql';
import {
IsDateString,
IsEnum,
IsNotEmpty,
IsString,
IsUUID,
} from 'class-validator';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
import { ServerlessFunctionSyncStatus } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
registerEnumType(ServerlessFunctionSyncStatus, {
name: 'ServerlessFunctionSyncStatus',
description: 'SyncStatus of the serverlessFunction',
});
@ObjectType('serverlessFunction')
@Authorize({
authorize: (context: any) => ({
workspaceId: { eq: context?.req?.user?.workspace?.id },
}),
})
@QueryOptions({
defaultResultSize: 10,
maxResultsSize: 1000,
})
export class ServerlessFunctionDto {
@IsUUID()
@IsNotEmpty()
@IDField(() => UUIDScalarType)
id: string;
@IsString()
@IsNotEmpty()
@Field()
name: string;
@IsString()
@IsNotEmpty()
@Field()
sourceCodeHash: string;
@IsEnum(ServerlessFunctionSyncStatus)
@IsNotEmpty()
@Field(() => ServerlessFunctionSyncStatus)
syncStatus: ServerlessFunctionSyncStatus;
@HideField()
workspaceId: string;
@IsDateString()
@Field()
createdAt: Date;
@IsDateString()
@Field()
updatedAt: Date;
}

View File

@@ -0,0 +1,43 @@
import {
Column,
CreateDateColumn,
Entity,
PrimaryGeneratedColumn,
Unique,
UpdateDateColumn,
} from 'typeorm';
export enum ServerlessFunctionSyncStatus {
NOT_READY = 'NOT_READY',
READY = 'READY',
}
@Entity('serverlessFunction')
@Unique('IndexOnNameAndWorkspaceIdUnique', ['name', 'workspaceId'])
export class ServerlessFunctionEntity {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: false })
name: string;
@Column({ nullable: false })
sourceCodeHash: string;
@Column({
nullable: false,
default: ServerlessFunctionSyncStatus.NOT_READY,
type: 'enum',
enum: ServerlessFunctionSyncStatus,
})
syncStatus: ServerlessFunctionSyncStatus;
@Column({ nullable: false, type: 'uuid' })
workspaceId: string;
@CreateDateColumn({ type: 'timestamptz' })
createdAt: Date;
@UpdateDateColumn({ type: 'timestamptz' })
updatedAt: Date;
}

View File

@@ -0,0 +1,14 @@
import { CustomException } from 'src/utils/custom-exception';
export class ServerlessFunctionException extends CustomException {
code: ServerlessFunctionExceptionCode;
constructor(message: string, code: ServerlessFunctionExceptionCode) {
super(message, code);
}
}
export enum ServerlessFunctionExceptionCode {
SERVERLESS_FUNCTION_NOT_FOUND = 'SERVERLESS_FUNCTION_NOT_FOUND',
SERVERLESS_FUNCTION_ALREADY_EXIST = 'SERVERLESS_FUNCTION_ALREADY_EXIST',
SERVERLESS_FUNCTION_NOT_READY = 'SERVERLESS_FUNCTION_NOT_READY',
}

View File

@@ -0,0 +1,50 @@
import { Module } from '@nestjs/common';
import {
NestjsQueryGraphQLModule,
PagingStrategies,
} from '@ptc-org/nestjs-query-graphql';
import { SortDirection } from '@ptc-org/nestjs-query-core';
import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { ServerlessModule } from 'src/engine/integrations/serverless/serverless.module';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { ServerlessFunctionResolver } from 'src/engine/metadata-modules/serverless-function/serverless-function.resolver';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { ServerlessFunctionDto } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto';
import { FileUploadModule } from 'src/engine/core-modules/file/file-upload/file-upload.module';
@Module({
imports: [
NestjsQueryGraphQLModule.forFeature({
imports: [
FileUploadModule,
NestjsQueryTypeOrmModule.forFeature(
[ServerlessFunctionEntity],
'metadata',
),
],
services: [ServerlessFunctionService],
resolvers: [
{
EntityClass: ServerlessFunctionEntity,
DTOClass: ServerlessFunctionDto,
ServiceClass: ServerlessFunctionService,
pagingStrategy: PagingStrategies.CURSOR,
read: {
defaultSort: [{ field: 'id', direction: SortDirection.DESC }],
},
create: { disabled: true },
update: { disabled: true },
delete: { disabled: true },
guards: [JwtAuthGuard],
},
],
}),
ServerlessModule,
],
providers: [ServerlessFunctionService, ServerlessFunctionResolver],
exports: [ServerlessFunctionService],
})
export class ServerlessFunctionModule {}

View File

@@ -0,0 +1,59 @@
import { UseGuards } from '@nestjs/common';
import { Args, Mutation, Resolver } from '@nestjs/graphql';
import { FileUpload, GraphQLUpload } from 'graphql-upload';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { ExecuteServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/execute-serverless-function.input';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { ServerlessFunctionDto } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto';
import { ServerlessFunctionExecutionResultDTO } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result-d-t.o';
import { serverlessFunctionGraphQLApiExceptionHandler } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-graphql-api-exception-handler.utils';
@UseGuards(JwtAuthGuard)
@Resolver()
export class ServerlessFunctionResolver {
constructor(
private readonly serverlessFunctionService: ServerlessFunctionService,
) {}
@Mutation(() => ServerlessFunctionDto)
async createOneServerlessFunction(
@Args({ name: 'file', type: () => GraphQLUpload })
file: FileUpload,
@Args('name', { type: () => String }) name: string,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
try {
return await this.serverlessFunctionService.createOne(
name,
workspaceId,
file,
);
} catch (error) {
serverlessFunctionGraphQLApiExceptionHandler(error);
}
}
@Mutation(() => ServerlessFunctionExecutionResultDTO)
async executeOneServerlessFunction(
@Args() executeServerlessFunctionInput: ExecuteServerlessFunctionInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
try {
const { name, payload } = executeServerlessFunctionInput;
return {
result: await this.serverlessFunctionService.executeOne(
name,
workspaceId,
payload,
),
};
} catch (error) {
serverlessFunctionGraphQLApiExceptionHandler(error);
}
}
}

View File

@@ -0,0 +1,112 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { join } from 'path';
import { FileUpload } from 'graphql-upload';
import { Repository } from 'typeorm';
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import { ServerlessService } from 'src/engine/integrations/serverless/serverless.service';
import {
ServerlessFunctionEntity,
ServerlessFunctionSyncStatus,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import {
ServerlessFunctionException,
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { readFileContent } from 'src/engine/integrations/file-storage/utils/read-file-content';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { SOURCE_FILE_NAME } from 'src/engine/integrations/serverless/drivers/constants/source-file-name';
import { serverlessFunctionCreateHash } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-create-hash.utils';
@Injectable()
export class ServerlessFunctionService {
constructor(
private readonly fileStorageService: FileStorageService,
private readonly serverlessService: ServerlessService,
@InjectRepository(ServerlessFunctionEntity, 'metadata')
private readonly serverlessFunctionRepository: Repository<ServerlessFunctionEntity>,
) {}
async executeOne(
name: string,
workspaceId: string,
payload: object | undefined = undefined,
) {
const functionToExecute = await this.serverlessFunctionRepository.findOne({
where: {
name,
workspaceId,
},
});
if (!functionToExecute) {
throw new ServerlessFunctionException(
`Function does not exist`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
}
if (
functionToExecute.syncStatus === ServerlessFunctionSyncStatus.NOT_READY
) {
throw new ServerlessFunctionException(
`Function is not ready to be executed`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
}
return this.serverlessService.execute(functionToExecute, payload);
}
async createOne(
name: string,
workspaceId: string,
{ createReadStream, mimetype }: FileUpload,
) {
const existingServerlessFunction =
await this.serverlessFunctionRepository.findOne({
where: { name, workspaceId },
});
if (existingServerlessFunction) {
throw new ServerlessFunctionException(
`Function already exists`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_ALREADY_EXIST,
);
}
const typescriptCode = await readFileContent(createReadStream());
const serverlessFunction = await this.serverlessFunctionRepository.save({
name,
workspaceId,
sourceCodeHash: serverlessFunctionCreateHash(typescriptCode),
});
const fileFolder = join(
FileFolder.ServerlessFunction,
workspaceId,
serverlessFunction.id,
);
await this.fileStorageService.write({
file: typescriptCode,
name: SOURCE_FILE_NAME,
mimeType: mimetype,
folder: fileFolder,
});
await this.serverlessService.build(serverlessFunction);
await this.serverlessFunctionRepository.update(serverlessFunction.id, {
syncStatus: ServerlessFunctionSyncStatus.READY,
});
return await this.serverlessFunctionRepository.findOneByOrFail({
id: serverlessFunction.id,
});
}
}

View File

@@ -0,0 +1,8 @@
import { createHash } from 'crypto';
export const serverlessFunctionCreateHash = (fileContent: string) => {
return createHash('sha512')
.update(fileContent)
.digest('hex')
.substring(0, 32);
};

View File

@@ -0,0 +1,26 @@
import {
ServerlessFunctionException,
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import {
ConflictError,
ForbiddenError,
InternalServerError,
NotFoundError,
} from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
export const serverlessFunctionGraphQLApiExceptionHandler = (error: any) => {
if (error instanceof ServerlessFunctionException) {
switch (error.code) {
case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND:
throw new NotFoundError(error.message);
case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_ALREADY_EXIST:
throw new ConflictError(error.message);
case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_READY:
throw new ForbiddenError(error.message);
default:
throw new InternalServerError(error.message);
}
}
throw error;
};

View File

@@ -147,6 +147,16 @@ yarn command:prod cron:calendar:calendar-event-list-fetch
['STORAGE_LOCAL_PATH', '.local-storage', 'data path (local storage)'],
]}></ArticleTable>
### Custom Code Execution
<ArticleTable options={[
['SERVERLESS_TYPE', 'local', "Serverless driver type: 'local' or 'lambda'"],
['SERVERLESS_LAMBDA_REGION', '', 'Lambda Region'],
['SERVERLESS_LAMBDA_ROLE', '', 'Lambda Role'],
['SERVERLESS_LAMBDA_ACCESS_KEY_ID', '', 'Optional depending on the authentication method'],
['SERVERLESS_LAMBDA_SECRET_ACCESS_KEY', '', 'Optional depending on the authentication method'],
]}></ArticleTable>
### Message Queue
<ArticleTable options={[

1281
yarn.lock

File diff suppressed because it is too large Load Diff