mirror of
https://github.com/lingble/twenty.git
synced 2025-11-02 05:37:56 +00:00
6181 workflows create a custom code executor (#6235)
Closes #6181 ## Testing - download Altair graphql dev tool https://altairgraphql.dev/#download - create a file locally `test.ts` containing: ``` export const handler = async (event: object, context: object) => { return { test: 'toto', data: event['data'] }; } ``` - play those requests in Altair: mutation UpsertFunction($file: Upload!) { upsertFunction(name: "toto", file: $file) } mutation ExecFunction { executeFunction(name:"toto", payload: {data: "titi"}) } - it will run the local driver, add those env variable to test with lambda driver ``` CUSTOM_CODE_ENGINE_DRIVER_TYPE=lambda LAMBDA_REGION=eu-west-2 LAMBDA_ROLE=<ASK_ME> ```
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
"@air/react-drag-to-select": "^5.0.8",
|
||||
"@apollo/client": "^3.7.17",
|
||||
"@apollo/server": "^4.7.3",
|
||||
"@aws-sdk/client-lambda": "^3.614.0",
|
||||
"@aws-sdk/client-s3": "^3.363.0",
|
||||
"@aws-sdk/credential-providers": "^3.363.0",
|
||||
"@blocknote/core": "^0.12.1",
|
||||
@@ -70,6 +71,8 @@
|
||||
"afterframe": "^1.0.2",
|
||||
"apollo-server-express": "^3.12.0",
|
||||
"apollo-upload-client": "^17.0.0",
|
||||
"archiver": "^7.0.1",
|
||||
"aws-sdk": "^2.1658.0",
|
||||
"axios": "^1.6.2",
|
||||
"bcrypt": "^5.1.1",
|
||||
"better-sqlite3": "^9.2.2",
|
||||
@@ -93,6 +96,7 @@
|
||||
"facepaint": "^1.2.1",
|
||||
"file-type": "16.5.4",
|
||||
"framer-motion": "^10.12.17",
|
||||
"fs-extra": "^11.2.0",
|
||||
"googleapis": "105",
|
||||
"graphiql": "^3.1.1",
|
||||
"graphql": "16.8.0",
|
||||
|
||||
@@ -26,6 +26,13 @@ export type Scalars = {
|
||||
Upload: { input: any; output: any; }
|
||||
};
|
||||
|
||||
export type AisqlQueryResult = {
|
||||
__typename?: 'AISQLQueryResult';
|
||||
queryFailedErrorMessage?: Maybe<Scalars['String']['output']>;
|
||||
sqlQuery: Scalars['String']['output'];
|
||||
sqlQueryResult?: Maybe<Scalars['String']['output']>;
|
||||
};
|
||||
|
||||
export type ActivateWorkspaceInput = {
|
||||
displayName?: InputMaybe<Scalars['String']['input']>;
|
||||
};
|
||||
@@ -345,6 +352,7 @@ export enum FileFolder {
|
||||
Attachment = 'Attachment',
|
||||
PersonPicture = 'PersonPicture',
|
||||
ProfilePicture = 'ProfilePicture',
|
||||
ServerlessFunction = 'ServerlessFunction',
|
||||
WorkspaceLogo = 'WorkspaceLogo'
|
||||
}
|
||||
|
||||
@@ -397,6 +405,7 @@ export type Mutation = {
|
||||
createOneObject: Object;
|
||||
createOneRelation: Relation;
|
||||
createOneRemoteServer: RemoteServer;
|
||||
createOneServerlessFunction: ServerlessFunction;
|
||||
deleteCurrentWorkspace: Workspace;
|
||||
deleteOneField: Field;
|
||||
deleteOneObject: Object;
|
||||
@@ -407,6 +416,7 @@ export type Mutation = {
|
||||
emailPasswordResetLink: EmailPasswordResetLink;
|
||||
enablePostgresProxy: PostgresCredentials;
|
||||
exchangeAuthorizationCode: ExchangeAuthCode;
|
||||
executeOneServerlessFunction: ServerlessFunctionExecutionResult;
|
||||
generateApiKeyToken: ApiKeyToken;
|
||||
generateJWT: AuthTokens;
|
||||
generateTransientToken: TransientToken;
|
||||
@@ -488,6 +498,12 @@ export type MutationCreateOneRemoteServerArgs = {
|
||||
};
|
||||
|
||||
|
||||
export type MutationCreateOneServerlessFunctionArgs = {
|
||||
file: Scalars['Upload']['input'];
|
||||
name: Scalars['String']['input'];
|
||||
};
|
||||
|
||||
|
||||
export type MutationDeleteOneFieldArgs = {
|
||||
input: DeleteOneFieldInput;
|
||||
};
|
||||
@@ -520,6 +536,12 @@ export type MutationExchangeAuthorizationCodeArgs = {
|
||||
};
|
||||
|
||||
|
||||
export type MutationExecuteOneServerlessFunctionArgs = {
|
||||
name: Scalars['String']['input'];
|
||||
payload?: InputMaybe<Scalars['JSON']['input']>;
|
||||
};
|
||||
|
||||
|
||||
export type MutationGenerateApiKeyTokenArgs = {
|
||||
apiKeyId: Scalars['String']['input'];
|
||||
expiresAt: Scalars['String']['input'];
|
||||
@@ -707,6 +729,7 @@ export type Query = {
|
||||
findManyRemoteServersByType: Array<RemoteServer>;
|
||||
findOneRemoteServerById: RemoteServer;
|
||||
findWorkspaceFromInviteHash: Workspace;
|
||||
getAISQLQuery: AisqlQueryResult;
|
||||
getPostgresCredentials?: Maybe<PostgresCredentials>;
|
||||
getProductPrices: ProductPricesEntity;
|
||||
getTimelineCalendarEventsFromCompanyId: TimelineCalendarEventsWithTotal;
|
||||
@@ -717,6 +740,8 @@ export type Query = {
|
||||
objects: ObjectConnection;
|
||||
relation: Relation;
|
||||
relations: RelationConnection;
|
||||
serverlessFunction: ServerlessFunction;
|
||||
serverlessFunctions: ServerlessFunctionConnection;
|
||||
validatePasswordResetToken: ValidatePasswordResetToken;
|
||||
};
|
||||
|
||||
@@ -768,6 +793,11 @@ export type QueryFindWorkspaceFromInviteHashArgs = {
|
||||
};
|
||||
|
||||
|
||||
export type QueryGetAisqlQueryArgs = {
|
||||
text: Scalars['String']['input'];
|
||||
};
|
||||
|
||||
|
||||
export type QueryGetProductPricesArgs = {
|
||||
product: Scalars['String']['input'];
|
||||
};
|
||||
@@ -822,6 +852,18 @@ export type QueryRelationsArgs = {
|
||||
};
|
||||
|
||||
|
||||
export type QueryServerlessFunctionArgs = {
|
||||
id: Scalars['UUID']['input'];
|
||||
};
|
||||
|
||||
|
||||
export type QueryServerlessFunctionsArgs = {
|
||||
filter?: ServerlessFunctionFilter;
|
||||
paging?: CursorPaging;
|
||||
sorting?: Array<ServerlessFunctionSort>;
|
||||
};
|
||||
|
||||
|
||||
export type QueryValidatePasswordResetTokenArgs = {
|
||||
passwordResetToken: Scalars['String']['input'];
|
||||
};
|
||||
@@ -915,6 +957,26 @@ export type Sentry = {
|
||||
release?: Maybe<Scalars['String']['output']>;
|
||||
};
|
||||
|
||||
export type ServerlessFunctionConnection = {
|
||||
__typename?: 'ServerlessFunctionConnection';
|
||||
/** Array of edges. */
|
||||
edges: Array<ServerlessFunctionEdge>;
|
||||
/** Paging information */
|
||||
pageInfo: PageInfo;
|
||||
};
|
||||
|
||||
export type ServerlessFunctionExecutionResult = {
|
||||
__typename?: 'ServerlessFunctionExecutionResult';
|
||||
/** Execution result in JSON format */
|
||||
result: Scalars['JSON']['output'];
|
||||
};
|
||||
|
||||
/** SyncStatus of the serverlessFunction */
|
||||
export enum ServerlessFunctionSyncStatus {
|
||||
NotReady = 'NOT_READY',
|
||||
Ready = 'READY'
|
||||
}
|
||||
|
||||
export type SessionEntity = {
|
||||
__typename?: 'SessionEntity';
|
||||
url?: Maybe<Scalars['String']['output']>;
|
||||
@@ -1345,6 +1407,39 @@ export type RelationEdge = {
|
||||
node: Relation;
|
||||
};
|
||||
|
||||
export type ServerlessFunction = {
|
||||
__typename?: 'serverlessFunction';
|
||||
createdAt: Scalars['DateTime']['output'];
|
||||
id: Scalars['UUID']['output'];
|
||||
name: Scalars['String']['output'];
|
||||
syncStatus: ServerlessFunctionSyncStatus;
|
||||
updatedAt: Scalars['DateTime']['output'];
|
||||
};
|
||||
|
||||
export type ServerlessFunctionEdge = {
|
||||
__typename?: 'serverlessFunctionEdge';
|
||||
/** Cursor for this node. */
|
||||
cursor: Scalars['ConnectionCursor']['output'];
|
||||
/** The node containing the serverlessFunction */
|
||||
node: ServerlessFunction;
|
||||
};
|
||||
|
||||
export type ServerlessFunctionFilter = {
|
||||
and?: InputMaybe<Array<ServerlessFunctionFilter>>;
|
||||
id?: InputMaybe<UuidFilterComparison>;
|
||||
or?: InputMaybe<Array<ServerlessFunctionFilter>>;
|
||||
};
|
||||
|
||||
export type ServerlessFunctionSort = {
|
||||
direction: SortDirection;
|
||||
field: ServerlessFunctionSortFields;
|
||||
nulls?: InputMaybe<SortNulls>;
|
||||
};
|
||||
|
||||
export enum ServerlessFunctionSortFields {
|
||||
Id = 'id'
|
||||
}
|
||||
|
||||
export type RemoteServerFieldsFragment = { __typename?: 'RemoteServer', id: string, createdAt: any, foreignDataWrapperId: string, foreignDataWrapperOptions?: any | null, foreignDataWrapperType: string, updatedAt: any, schema?: string | null, label: string, userMappingOptions?: { __typename?: 'UserMappingOptionsUser', user?: string | null } | null };
|
||||
|
||||
export type RemoteTableFieldsFragment = { __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<DistantTableUpdate> | null };
|
||||
|
||||
@@ -258,6 +258,7 @@ export enum FileFolder {
|
||||
Attachment = 'Attachment',
|
||||
PersonPicture = 'PersonPicture',
|
||||
ProfilePicture = 'ProfilePicture',
|
||||
ServerlessFunction = 'ServerlessFunction',
|
||||
WorkspaceLogo = 'WorkspaceLogo'
|
||||
}
|
||||
|
||||
@@ -683,6 +684,26 @@ export type Sentry = {
|
||||
release?: Maybe<Scalars['String']>;
|
||||
};
|
||||
|
||||
export type ServerlessFunctionConnection = {
|
||||
__typename?: 'ServerlessFunctionConnection';
|
||||
/** Array of edges. */
|
||||
edges: Array<ServerlessFunctionEdge>;
|
||||
/** Paging information */
|
||||
pageInfo: PageInfo;
|
||||
};
|
||||
|
||||
export type ServerlessFunctionExecutionResult = {
|
||||
__typename?: 'ServerlessFunctionExecutionResult';
|
||||
/** Execution result in JSON format */
|
||||
result: Scalars['JSON'];
|
||||
};
|
||||
|
||||
/** SyncStatus of the serverlessFunction */
|
||||
export enum ServerlessFunctionSyncStatus {
|
||||
NotReady = 'NOT_READY',
|
||||
Ready = 'READY'
|
||||
}
|
||||
|
||||
export type SessionEntity = {
|
||||
__typename?: 'SessionEntity';
|
||||
url?: Maybe<Scalars['String']>;
|
||||
@@ -1064,6 +1085,23 @@ export type RelationEdge = {
|
||||
node: Relation;
|
||||
};
|
||||
|
||||
export type ServerlessFunction = {
|
||||
__typename?: 'serverlessFunction';
|
||||
createdAt: Scalars['DateTime'];
|
||||
id: Scalars['UUID'];
|
||||
name: Scalars['String'];
|
||||
syncStatus: ServerlessFunctionSyncStatus;
|
||||
updatedAt: Scalars['DateTime'];
|
||||
};
|
||||
|
||||
export type ServerlessFunctionEdge = {
|
||||
__typename?: 'serverlessFunctionEdge';
|
||||
/** Cursor for this node. */
|
||||
cursor: Scalars['ConnectionCursor'];
|
||||
/** The node containing the serverlessFunction */
|
||||
node: ServerlessFunction;
|
||||
};
|
||||
|
||||
export type TimelineCalendarEventFragmentFragment = { __typename?: 'TimelineCalendarEvent', id: any, title: string, description: string, location: string, startsAt: string, endsAt: string, isFullDay: boolean, visibility: CalendarChannelVisibility, participants: Array<{ __typename?: 'TimelineCalendarEventParticipant', personId?: any | null, workspaceMemberId?: any | null, firstName: string, lastName: string, displayName: string, avatarUrl: string, handle: string }> };
|
||||
|
||||
export type TimelineCalendarEventParticipantFragmentFragment = { __typename?: 'TimelineCalendarEventParticipant', personId?: any | null, workspaceMemberId?: any | null, firstName: string, lastName: string, displayName: string, avatarUrl: string, handle: string };
|
||||
|
||||
@@ -35,6 +35,7 @@ SIGN_IN_PREFILLED=true
|
||||
# AUTH_GOOGLE_CLIENT_SECRET=replace_me_with_google_client_secret
|
||||
# AUTH_GOOGLE_CALLBACK_URL=http://localhost:3000/auth/google/redirect
|
||||
# AUTH_GOOGLE_APIS_CALLBACK_URL=http://localhost:3000/auth/google-apis/get-access-token
|
||||
# SERVERLESS_TYPE=local
|
||||
# STORAGE_TYPE=local
|
||||
# STORAGE_LOCAL_PATH=.local-storage
|
||||
# SUPPORT_DRIVER=front
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class CreateServerlessFunctionTable1721210534680
|
||||
implements MigrationInterface
|
||||
{
|
||||
name = 'CreateServerlessFunctionTable1721210534680';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`CREATE TYPE "metadata"."serverlessFunction_syncstatus_enum" AS ENUM('NOT_READY', 'READY')`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE TABLE "metadata"."serverlessFunction" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "sourceCodeHash" character varying NOT NULL, "syncStatus" "metadata"."serverlessFunction_syncstatus_enum" NOT NULL DEFAULT 'NOT_READY', "workspaceId" uuid NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "IndexOnNameAndWorkspaceIdUnique" UNIQUE ("name", "workspaceId"), CONSTRAINT "PK_49bfacee064bee9d0d486483b60" PRIMARY KEY ("id"))`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`DROP TABLE "metadata"."serverlessFunction"`);
|
||||
await queryRunner.query(
|
||||
`DROP TYPE "metadata"."serverlessFunction_syncstatus_enum"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -13,9 +13,9 @@ import { HealthModule } from 'src/engine/core-modules/health/health.module';
|
||||
import { AISQLQueryModule } from 'src/engine/core-modules/ai-sql-query/ai-sql-query.module';
|
||||
import { PostgresCredentialsModule } from 'src/engine/core-modules/postgres-credentials/postgres-credentials.module';
|
||||
|
||||
import { AnalyticsModule } from './analytics/analytics.module';
|
||||
import { FileModule } from './file/file.module';
|
||||
import { ClientConfigModule } from './client-config/client-config.module';
|
||||
import { FileModule } from './file/file.module';
|
||||
import { AnalyticsModule } from './analytics/analytics.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
|
||||
@@ -5,6 +5,7 @@ export enum FileFolder {
|
||||
WorkspaceLogo = 'workspace-logo',
|
||||
Attachment = 'attachment',
|
||||
PersonPicture = 'person-picture',
|
||||
ServerlessFunction = 'serverless-function',
|
||||
}
|
||||
|
||||
registerEnumType(FileFolder, {
|
||||
|
||||
@@ -20,6 +20,7 @@ import { NodeEnvironment } from 'src/engine/integrations/environment/interfaces/
|
||||
import { LLMChatModelDriver } from 'src/engine/integrations/llm-chat-model/interfaces/llm-chat-model.interface';
|
||||
import { LLMTracingDriver } from 'src/engine/integrations/llm-tracing/interfaces/llm-tracing.interface';
|
||||
|
||||
import { ServerlessDriverType } from 'src/engine/integrations/serverless/serverless.interface';
|
||||
import { assert } from 'src/utils/assert';
|
||||
import { CastToStringArray } from 'src/engine/integrations/environment/decorators/cast-to-string-array.decorator';
|
||||
import { ExceptionHandlerDriver } from 'src/engine/integrations/exception-handler/interfaces';
|
||||
@@ -204,6 +205,30 @@ export class EnvironmentVariables {
|
||||
@ValidateIf((env) => env.AUTH_GOOGLE_ENABLED)
|
||||
AUTH_GOOGLE_CALLBACK_URL: string;
|
||||
|
||||
// Custom Code Engine
|
||||
@IsEnum(ServerlessDriverType)
|
||||
@IsOptional()
|
||||
SERVERLESS_TYPE: ServerlessDriverType = ServerlessDriverType.Local;
|
||||
|
||||
@ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda)
|
||||
@IsAWSRegion()
|
||||
SERVERLESS_LAMBDA_REGION: AwsRegion;
|
||||
|
||||
@ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda)
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
SERVERLESS_LAMBDA_ROLE: string;
|
||||
|
||||
@ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda)
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
SERVERLESS_LAMBDA_ACCESS_KEY_ID: string;
|
||||
|
||||
@ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda)
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
SERVERLESS_LAMBDA_SECRET_ACCESS_KEY: string;
|
||||
|
||||
// Storage
|
||||
@IsEnum(StorageDriverType)
|
||||
@IsOptional()
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { Readable } from 'stream';
|
||||
|
||||
export const readFileContent = async (stream: Readable): Promise<string> => {
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
}
|
||||
|
||||
return Buffer.concat(chunks).toString('utf8');
|
||||
};
|
||||
@@ -16,6 +16,10 @@ import { LLMChatModelModule } from 'src/engine/integrations/llm-chat-model/llm-c
|
||||
import { llmChatModelModuleFactory } from 'src/engine/integrations/llm-chat-model/llm-chat-model.module-factory';
|
||||
import { LLMTracingModule } from 'src/engine/integrations/llm-tracing/llm-tracing.module';
|
||||
import { llmTracingModuleFactory } from 'src/engine/integrations/llm-tracing/llm-tracing.module-factory';
|
||||
import { ServerlessModule } from 'src/engine/integrations/serverless/serverless.module';
|
||||
import { serverlessModuleFactory } from 'src/engine/integrations/serverless/serverless-module.factory';
|
||||
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
|
||||
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
|
||||
|
||||
import { EnvironmentModule } from './environment/environment.module';
|
||||
import { EnvironmentService } from './environment/environment.service';
|
||||
@@ -62,6 +66,14 @@ import { MessageQueueModule } from './message-queue/message-queue.module';
|
||||
useFactory: llmTracingModuleFactory,
|
||||
inject: [EnvironmentService],
|
||||
}),
|
||||
ServerlessModule.forRootAsync({
|
||||
useFactory: serverlessModuleFactory,
|
||||
inject: [
|
||||
EnvironmentService,
|
||||
FileStorageService,
|
||||
BuildDirectoryManagerService,
|
||||
],
|
||||
}),
|
||||
],
|
||||
exports: [],
|
||||
providers: [],
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { join } from 'path';
|
||||
|
||||
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
|
||||
|
||||
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
|
||||
import { SOURCE_FILE_NAME } from 'src/engine/integrations/serverless/drivers/constants/source-file-name';
|
||||
import { readFileContent } from 'src/engine/integrations/file-storage/utils/read-file-content';
|
||||
import { compileTypescript } from 'src/engine/integrations/serverless/drivers/utils/compile-typescript';
|
||||
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
|
||||
|
||||
export class BaseServerlessDriver {
|
||||
getFolderPath(serverlessFunction: ServerlessFunctionEntity) {
|
||||
return join(
|
||||
FileFolder.ServerlessFunction,
|
||||
serverlessFunction.workspaceId,
|
||||
serverlessFunction.id,
|
||||
);
|
||||
}
|
||||
|
||||
async getCompiledCode(
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
fileStorageService: FileStorageService,
|
||||
) {
|
||||
const folderPath = this.getFolderPath(serverlessFunction);
|
||||
const fileStream = await fileStorageService.read({
|
||||
folderPath,
|
||||
filename: SOURCE_FILE_NAME,
|
||||
});
|
||||
const typescriptCode = await readFileContent(fileStream);
|
||||
|
||||
return compileTypescript(typescriptCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export const BUILD_FILE_NAME = 'build.js';
|
||||
@@ -0,0 +1 @@
|
||||
export const SOURCE_FILE_NAME = 'source.ts';
|
||||
@@ -0,0 +1,9 @@
|
||||
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
|
||||
|
||||
export interface ServerlessDriver {
|
||||
build(serverlessFunction: ServerlessFunctionEntity): Promise<void>;
|
||||
execute(
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
payload: object | undefined,
|
||||
): Promise<object>;
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import fs from 'fs';
|
||||
|
||||
import {
|
||||
CreateFunctionCommand,
|
||||
Lambda,
|
||||
LambdaClientConfig,
|
||||
InvokeCommand,
|
||||
} from '@aws-sdk/client-lambda';
|
||||
import { CreateFunctionCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/CreateFunctionCommand';
|
||||
|
||||
import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
|
||||
|
||||
import { createZipFile } from 'src/engine/integrations/serverless/drivers/utils/create-zip-file';
|
||||
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
|
||||
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
|
||||
import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
|
||||
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
|
||||
|
||||
export interface LambdaDriverOptions extends LambdaClientConfig {
|
||||
fileStorageService: FileStorageService;
|
||||
buildDirectoryManagerService: BuildDirectoryManagerService;
|
||||
region: string;
|
||||
role: string;
|
||||
}
|
||||
|
||||
export class LambdaDriver
|
||||
extends BaseServerlessDriver
|
||||
implements ServerlessDriver
|
||||
{
|
||||
private readonly lambdaClient: Lambda;
|
||||
private readonly lambdaRole: string;
|
||||
private readonly fileStorageService: FileStorageService;
|
||||
private readonly buildDirectoryManagerService: BuildDirectoryManagerService;
|
||||
|
||||
constructor(options: LambdaDriverOptions) {
|
||||
super();
|
||||
const { region, role, ...lambdaOptions } = options;
|
||||
|
||||
this.lambdaClient = new Lambda({ ...lambdaOptions, region });
|
||||
this.lambdaRole = role;
|
||||
this.fileStorageService = options.fileStorageService;
|
||||
this.buildDirectoryManagerService = options.buildDirectoryManagerService;
|
||||
}
|
||||
|
||||
async build(serverlessFunction: ServerlessFunctionEntity) {
|
||||
const javascriptCode = await this.getCompiledCode(
|
||||
serverlessFunction,
|
||||
this.fileStorageService,
|
||||
);
|
||||
|
||||
const {
|
||||
sourceTemporaryDir,
|
||||
lambdaZipPath,
|
||||
javascriptFilePath,
|
||||
lambdaHandler,
|
||||
} = await this.buildDirectoryManagerService.init();
|
||||
|
||||
await fs.promises.writeFile(javascriptFilePath, javascriptCode);
|
||||
|
||||
await createZipFile(sourceTemporaryDir, lambdaZipPath);
|
||||
|
||||
const params: CreateFunctionCommandInput = {
|
||||
Code: {
|
||||
ZipFile: await fs.promises.readFile(lambdaZipPath),
|
||||
},
|
||||
FunctionName: serverlessFunction.id,
|
||||
Handler: lambdaHandler,
|
||||
Role: this.lambdaRole,
|
||||
Runtime: 'nodejs18.x',
|
||||
Description: 'Lambda function to run user script',
|
||||
Timeout: 900,
|
||||
};
|
||||
|
||||
const command = new CreateFunctionCommand(params);
|
||||
|
||||
await this.lambdaClient.send(command);
|
||||
|
||||
await this.buildDirectoryManagerService.clean();
|
||||
}
|
||||
|
||||
async execute(
|
||||
functionToExecute: ServerlessFunctionEntity,
|
||||
payload: object | undefined = undefined,
|
||||
): Promise<object> {
|
||||
const params = {
|
||||
FunctionName: functionToExecute.id,
|
||||
Payload: JSON.stringify(payload),
|
||||
};
|
||||
|
||||
const command = new InvokeCommand(params);
|
||||
|
||||
const result = await this.lambdaClient.send(command);
|
||||
|
||||
if (!result.Payload) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return JSON.parse(result.Payload.transformToString());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { promises as fs } from 'fs';
|
||||
import { fork } from 'child_process';
|
||||
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
|
||||
|
||||
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
|
||||
import { readFileContent } from 'src/engine/integrations/file-storage/utils/read-file-content';
|
||||
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
|
||||
import { BUILD_FILE_NAME } from 'src/engine/integrations/serverless/drivers/constants/build-file-name';
|
||||
import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
|
||||
|
||||
export interface LocalDriverOptions {
|
||||
fileStorageService: FileStorageService;
|
||||
}
|
||||
|
||||
export class LocalDriver
|
||||
extends BaseServerlessDriver
|
||||
implements ServerlessDriver
|
||||
{
|
||||
private readonly fileStorageService: FileStorageService;
|
||||
|
||||
constructor(options: LocalDriverOptions) {
|
||||
super();
|
||||
this.fileStorageService = options.fileStorageService;
|
||||
}
|
||||
|
||||
async build(serverlessFunction: ServerlessFunctionEntity) {
|
||||
const javascriptCode = await this.getCompiledCode(
|
||||
serverlessFunction,
|
||||
this.fileStorageService,
|
||||
);
|
||||
|
||||
await this.fileStorageService.write({
|
||||
file: javascriptCode,
|
||||
name: BUILD_FILE_NAME,
|
||||
mimeType: undefined,
|
||||
folder: this.getFolderPath(serverlessFunction),
|
||||
});
|
||||
}
|
||||
|
||||
async execute(
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
payload: object | undefined = undefined,
|
||||
): Promise<object> {
|
||||
const fileStream = await this.fileStorageService.read({
|
||||
folderPath: this.getFolderPath(serverlessFunction),
|
||||
filename: BUILD_FILE_NAME,
|
||||
});
|
||||
const fileContent = await readFileContent(fileStream);
|
||||
|
||||
const tmpFilePath = join(tmpdir(), `${v4()}.js`);
|
||||
|
||||
const modifiedContent = `
|
||||
process.on('message', async (message) => {
|
||||
const { event, context } = message;
|
||||
const result = await handler(event, context);
|
||||
process.send(result);
|
||||
});
|
||||
|
||||
${fileContent}
|
||||
`;
|
||||
|
||||
await fs.writeFile(tmpFilePath, modifiedContent);
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
const child = fork(tmpFilePath);
|
||||
|
||||
child.on('message', (message: object) => {
|
||||
resolve(message);
|
||||
child.kill();
|
||||
fs.unlink(tmpFilePath);
|
||||
});
|
||||
|
||||
child.on('error', (error) => {
|
||||
reject(error);
|
||||
child.kill();
|
||||
fs.unlink(tmpFilePath);
|
||||
});
|
||||
|
||||
child.on('exit', (code) => {
|
||||
if (code && code !== 0) {
|
||||
reject(new Error(`Child process exited with code ${code}`));
|
||||
fs.unlink(tmpFilePath);
|
||||
}
|
||||
});
|
||||
|
||||
child.send({ event: payload });
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import fs from 'fs';
|
||||
|
||||
import fsExtra from 'fs-extra';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
const TEMPORARY_LAMBDA_FOLDER = 'twenty-build-lambda-temp-folder';
|
||||
const TEMPORARY_LAMBDA_SOURCE_FOLDER = 'src';
|
||||
const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
|
||||
const LAMBDA_ENTRY_FILE_NAME = 'index.js';
|
||||
|
||||
@Injectable()
|
||||
export class BuildDirectoryManagerService {
|
||||
private temporaryDir = join(tmpdir(), `${TEMPORARY_LAMBDA_FOLDER}_${v4()}`);
|
||||
private lambdaHandler = `${LAMBDA_ENTRY_FILE_NAME.split('.')[0]}.handler`;
|
||||
|
||||
async init() {
|
||||
const sourceTemporaryDir = join(
|
||||
this.temporaryDir,
|
||||
TEMPORARY_LAMBDA_SOURCE_FOLDER,
|
||||
);
|
||||
const lambdaZipPath = join(this.temporaryDir, LAMBDA_ZIP_FILE_NAME);
|
||||
const javascriptFilePath = join(sourceTemporaryDir, LAMBDA_ENTRY_FILE_NAME);
|
||||
|
||||
if (!fs.existsSync(this.temporaryDir)) {
|
||||
await fs.promises.mkdir(this.temporaryDir);
|
||||
await fs.promises.mkdir(sourceTemporaryDir);
|
||||
} else {
|
||||
await fsExtra.emptyDir(this.temporaryDir);
|
||||
await fs.promises.mkdir(sourceTemporaryDir);
|
||||
}
|
||||
|
||||
return {
|
||||
sourceTemporaryDir,
|
||||
lambdaZipPath,
|
||||
javascriptFilePath,
|
||||
lambdaHandler: this.lambdaHandler,
|
||||
};
|
||||
}
|
||||
|
||||
async clean() {
|
||||
await fsExtra.emptyDir(this.temporaryDir);
|
||||
await fs.promises.rmdir(this.temporaryDir);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import ts from 'typescript';
|
||||
|
||||
export const compileTypescript = (typescriptCode: string): string => {
|
||||
const options: ts.CompilerOptions = {
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
target: ts.ScriptTarget.ES2017,
|
||||
moduleResolution: ts.ModuleResolutionKind.Node10,
|
||||
esModuleInterop: true,
|
||||
resolveJsonModule: true,
|
||||
allowSyntheticDefaultImports: true,
|
||||
types: ['node'],
|
||||
};
|
||||
|
||||
const result = ts.transpileModule(typescriptCode, {
|
||||
compilerOptions: options,
|
||||
});
|
||||
|
||||
return result.outputText;
|
||||
};
|
||||
@@ -0,0 +1,21 @@
|
||||
import fs from 'fs';
|
||||
import { pipeline } from 'stream/promises';
|
||||
|
||||
import archiver from 'archiver';
|
||||
|
||||
export const createZipFile = async (
|
||||
sourceDir: string,
|
||||
outPath: string,
|
||||
): Promise<void> => {
|
||||
const output = fs.createWriteStream(outPath);
|
||||
const archive = archiver('zip', {
|
||||
zlib: { level: 9 }, // Compression level
|
||||
});
|
||||
|
||||
const p = pipeline(archive, output);
|
||||
|
||||
archive.directory(sourceDir, false);
|
||||
archive.finalize();
|
||||
|
||||
return p;
|
||||
};
|
||||
@@ -0,0 +1,59 @@
|
||||
import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
|
||||
|
||||
import {
|
||||
ServerlessModuleOptions,
|
||||
ServerlessDriverType,
|
||||
} from 'src/engine/integrations/serverless/serverless.interface';
|
||||
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
|
||||
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
|
||||
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
|
||||
|
||||
export const serverlessModuleFactory = async (
|
||||
environmentService: EnvironmentService,
|
||||
fileStorageService: FileStorageService,
|
||||
buildDirectoryManagerService: BuildDirectoryManagerService,
|
||||
): Promise<ServerlessModuleOptions> => {
|
||||
const driverType = environmentService.get('SERVERLESS_TYPE');
|
||||
const options = { fileStorageService };
|
||||
|
||||
switch (driverType) {
|
||||
case ServerlessDriverType.Local: {
|
||||
return {
|
||||
type: ServerlessDriverType.Local,
|
||||
options,
|
||||
};
|
||||
}
|
||||
case ServerlessDriverType.Lambda: {
|
||||
const region = environmentService.get('SERVERLESS_LAMBDA_REGION');
|
||||
const accessKeyId = environmentService.get(
|
||||
'SERVERLESS_LAMBDA_ACCESS_KEY_ID',
|
||||
);
|
||||
const secretAccessKey = environmentService.get(
|
||||
'SERVERLESS_LAMBDA_SECRET_ACCESS_KEY',
|
||||
);
|
||||
const role = environmentService.get('SERVERLESS_LAMBDA_ROLE');
|
||||
|
||||
return {
|
||||
type: ServerlessDriverType.Lambda,
|
||||
options: {
|
||||
...options,
|
||||
buildDirectoryManagerService,
|
||||
credentials: accessKeyId
|
||||
? {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
}
|
||||
: fromNodeProviderChain({
|
||||
clientConfig: { region },
|
||||
}),
|
||||
region: region ?? '',
|
||||
role: role ?? '',
|
||||
},
|
||||
};
|
||||
}
|
||||
default:
|
||||
throw new Error(
|
||||
`Invalid serverless driver type (${driverType}), check your .env file`,
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1 @@
|
||||
export const SERVERLESS_DRIVER = Symbol('SERVERLESS_DRIVER');
|
||||
@@ -0,0 +1,30 @@
|
||||
import { FactoryProvider, ModuleMetadata } from '@nestjs/common';
|
||||
|
||||
import { LocalDriverOptions } from 'src/engine/integrations/serverless/drivers/local.driver';
|
||||
import { LambdaDriverOptions } from 'src/engine/integrations/serverless/drivers/lambda.driver';
|
||||
|
||||
export enum ServerlessDriverType {
|
||||
Lambda = 'lambda',
|
||||
Local = 'local',
|
||||
}
|
||||
|
||||
export interface LocalDriverFactoryOptions {
|
||||
type: ServerlessDriverType.Local;
|
||||
options: LocalDriverOptions;
|
||||
}
|
||||
|
||||
export interface LambdaDriverFactoryOptions {
|
||||
type: ServerlessDriverType.Lambda;
|
||||
options: LambdaDriverOptions;
|
||||
}
|
||||
|
||||
export type ServerlessModuleOptions =
|
||||
| LocalDriverFactoryOptions
|
||||
| LambdaDriverFactoryOptions;
|
||||
|
||||
export type ServerlessModuleAsyncOptions = {
|
||||
useFactory: (
|
||||
...args: any[]
|
||||
) => ServerlessModuleOptions | Promise<ServerlessModuleOptions>;
|
||||
} & Pick<ModuleMetadata, 'imports'> &
|
||||
Pick<FactoryProvider, 'inject'>;
|
||||
@@ -0,0 +1,35 @@
|
||||
import { DynamicModule, Global } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
ServerlessDriverType,
|
||||
ServerlessModuleAsyncOptions,
|
||||
} from 'src/engine/integrations/serverless/serverless.interface';
|
||||
import { ServerlessService } from 'src/engine/integrations/serverless/serverless.service';
|
||||
import { SERVERLESS_DRIVER } from 'src/engine/integrations/serverless/serverless.constants';
|
||||
import { LocalDriver } from 'src/engine/integrations/serverless/drivers/local.driver';
|
||||
import { LambdaDriver } from 'src/engine/integrations/serverless/drivers/lambda.driver';
|
||||
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
|
||||
|
||||
@Global()
|
||||
export class ServerlessModule {
|
||||
static forRootAsync(options: ServerlessModuleAsyncOptions): DynamicModule {
|
||||
const provider = {
|
||||
provide: SERVERLESS_DRIVER,
|
||||
useFactory: async (...args: any[]) => {
|
||||
const config = await options.useFactory(...args);
|
||||
|
||||
return config?.type === ServerlessDriverType.Local
|
||||
? new LocalDriver(config.options)
|
||||
: new LambdaDriver(config.options);
|
||||
},
|
||||
inject: options.inject || [],
|
||||
};
|
||||
|
||||
return {
|
||||
module: ServerlessModule,
|
||||
imports: options.imports || [],
|
||||
providers: [ServerlessService, BuildDirectoryManagerService, provider],
|
||||
exports: [ServerlessService],
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
|
||||
import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
|
||||
|
||||
import { SERVERLESS_DRIVER } from 'src/engine/integrations/serverless/serverless.constants';
|
||||
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
|
||||
|
||||
@Injectable()
|
||||
export class ServerlessService implements ServerlessDriver {
|
||||
constructor(@Inject(SERVERLESS_DRIVER) private driver: ServerlessDriver) {}
|
||||
|
||||
async build(serverlessFunction: ServerlessFunctionEntity): Promise<void> {
|
||||
return this.driver.build(serverlessFunction);
|
||||
}
|
||||
|
||||
async execute(
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
payload: object | undefined = undefined,
|
||||
) {
|
||||
return this.driver.execute(serverlessFunction, payload);
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { RelationMetadataModule } from 'src/engine/metadata-modules/relation-met
|
||||
import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module';
|
||||
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
|
||||
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
|
||||
import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -14,6 +15,7 @@ import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-
|
||||
FieldMetadataModule,
|
||||
ObjectMetadataModule,
|
||||
RelationMetadataModule,
|
||||
ServerlessFunctionModule,
|
||||
WorkspaceCacheVersionModule,
|
||||
WorkspaceMigrationModule,
|
||||
RemoteServerModule,
|
||||
@@ -24,6 +26,7 @@ import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-
|
||||
FieldMetadataModule,
|
||||
ObjectMetadataModule,
|
||||
RelationMetadataModule,
|
||||
ServerlessFunctionModule,
|
||||
RemoteServerModule,
|
||||
],
|
||||
})
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { ArgsType, Field } from '@nestjs/graphql';
|
||||
|
||||
import { IsNotEmpty, IsObject, IsOptional, IsString } from 'class-validator';
|
||||
import graphqlTypeJson from 'graphql-type-json';
|
||||
|
||||
@ArgsType()
|
||||
export class ExecuteServerlessFunctionInput {
|
||||
@Field({ description: 'Name of the serverless function to execute' })
|
||||
@IsNotEmpty()
|
||||
@IsString()
|
||||
name: string;
|
||||
|
||||
@Field(() => graphqlTypeJson, {
|
||||
description: 'Payload in JSON format',
|
||||
nullable: true,
|
||||
})
|
||||
@IsObject()
|
||||
@IsOptional()
|
||||
payload?: JSON;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { IsObject } from 'class-validator';
|
||||
import graphqlTypeJson from 'graphql-type-json';
|
||||
|
||||
@ObjectType('ServerlessFunctionExecutionResult')
|
||||
export class ServerlessFunctionExecutionResultDTO {
|
||||
@IsObject()
|
||||
@Field(() => graphqlTypeJson, {
|
||||
description: 'Execution result in JSON format',
|
||||
})
|
||||
result: JSON;
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import {
|
||||
Field,
|
||||
HideField,
|
||||
ObjectType,
|
||||
registerEnumType,
|
||||
} from '@nestjs/graphql';
|
||||
|
||||
import {
|
||||
Authorize,
|
||||
IDField,
|
||||
QueryOptions,
|
||||
} from '@ptc-org/nestjs-query-graphql';
|
||||
import {
|
||||
IsDateString,
|
||||
IsEnum,
|
||||
IsNotEmpty,
|
||||
IsString,
|
||||
IsUUID,
|
||||
} from 'class-validator';
|
||||
|
||||
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
|
||||
import { ServerlessFunctionSyncStatus } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
|
||||
|
||||
registerEnumType(ServerlessFunctionSyncStatus, {
|
||||
name: 'ServerlessFunctionSyncStatus',
|
||||
description: 'SyncStatus of the serverlessFunction',
|
||||
});
|
||||
|
||||
@ObjectType('serverlessFunction')
|
||||
@Authorize({
|
||||
authorize: (context: any) => ({
|
||||
workspaceId: { eq: context?.req?.user?.workspace?.id },
|
||||
}),
|
||||
})
|
||||
@QueryOptions({
|
||||
defaultResultSize: 10,
|
||||
maxResultsSize: 1000,
|
||||
})
|
||||
export class ServerlessFunctionDto {
|
||||
@IsUUID()
|
||||
@IsNotEmpty()
|
||||
@IDField(() => UUIDScalarType)
|
||||
id: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
@Field()
|
||||
name: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
@Field()
|
||||
sourceCodeHash: string;
|
||||
|
||||
@IsEnum(ServerlessFunctionSyncStatus)
|
||||
@IsNotEmpty()
|
||||
@Field(() => ServerlessFunctionSyncStatus)
|
||||
syncStatus: ServerlessFunctionSyncStatus;
|
||||
|
||||
@HideField()
|
||||
workspaceId: string;
|
||||
|
||||
@IsDateString()
|
||||
@Field()
|
||||
createdAt: Date;
|
||||
|
||||
@IsDateString()
|
||||
@Field()
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Unique,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
|
||||
export enum ServerlessFunctionSyncStatus {
|
||||
NOT_READY = 'NOT_READY',
|
||||
READY = 'READY',
|
||||
}
|
||||
|
||||
@Entity('serverlessFunction')
|
||||
@Unique('IndexOnNameAndWorkspaceIdUnique', ['name', 'workspaceId'])
|
||||
export class ServerlessFunctionEntity {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ nullable: false })
|
||||
name: string;
|
||||
|
||||
@Column({ nullable: false })
|
||||
sourceCodeHash: string;
|
||||
|
||||
@Column({
|
||||
nullable: false,
|
||||
default: ServerlessFunctionSyncStatus.NOT_READY,
|
||||
type: 'enum',
|
||||
enum: ServerlessFunctionSyncStatus,
|
||||
})
|
||||
syncStatus: ServerlessFunctionSyncStatus;
|
||||
|
||||
@Column({ nullable: false, type: 'uuid' })
|
||||
workspaceId: string;
|
||||
|
||||
@CreateDateColumn({ type: 'timestamptz' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ type: 'timestamptz' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { CustomException } from 'src/utils/custom-exception';
|
||||
|
||||
export class ServerlessFunctionException extends CustomException {
|
||||
code: ServerlessFunctionExceptionCode;
|
||||
constructor(message: string, code: ServerlessFunctionExceptionCode) {
|
||||
super(message, code);
|
||||
}
|
||||
}
|
||||
|
||||
export enum ServerlessFunctionExceptionCode {
|
||||
SERVERLESS_FUNCTION_NOT_FOUND = 'SERVERLESS_FUNCTION_NOT_FOUND',
|
||||
SERVERLESS_FUNCTION_ALREADY_EXIST = 'SERVERLESS_FUNCTION_ALREADY_EXIST',
|
||||
SERVERLESS_FUNCTION_NOT_READY = 'SERVERLESS_FUNCTION_NOT_READY',
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
NestjsQueryGraphQLModule,
|
||||
PagingStrategies,
|
||||
} from '@ptc-org/nestjs-query-graphql';
|
||||
import { SortDirection } from '@ptc-org/nestjs-query-core';
|
||||
import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm';
|
||||
|
||||
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
|
||||
import { ServerlessModule } from 'src/engine/integrations/serverless/serverless.module';
|
||||
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
|
||||
import { ServerlessFunctionResolver } from 'src/engine/metadata-modules/serverless-function/serverless-function.resolver';
|
||||
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
|
||||
import { ServerlessFunctionDto } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto';
|
||||
import { FileUploadModule } from 'src/engine/core-modules/file/file-upload/file-upload.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
NestjsQueryGraphQLModule.forFeature({
|
||||
imports: [
|
||||
FileUploadModule,
|
||||
NestjsQueryTypeOrmModule.forFeature(
|
||||
[ServerlessFunctionEntity],
|
||||
'metadata',
|
||||
),
|
||||
],
|
||||
services: [ServerlessFunctionService],
|
||||
resolvers: [
|
||||
{
|
||||
EntityClass: ServerlessFunctionEntity,
|
||||
DTOClass: ServerlessFunctionDto,
|
||||
ServiceClass: ServerlessFunctionService,
|
||||
pagingStrategy: PagingStrategies.CURSOR,
|
||||
read: {
|
||||
defaultSort: [{ field: 'id', direction: SortDirection.DESC }],
|
||||
},
|
||||
create: { disabled: true },
|
||||
update: { disabled: true },
|
||||
delete: { disabled: true },
|
||||
guards: [JwtAuthGuard],
|
||||
},
|
||||
],
|
||||
}),
|
||||
ServerlessModule,
|
||||
],
|
||||
providers: [ServerlessFunctionService, ServerlessFunctionResolver],
|
||||
exports: [ServerlessFunctionService],
|
||||
})
|
||||
export class ServerlessFunctionModule {}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { UseGuards } from '@nestjs/common';
|
||||
import { Args, Mutation, Resolver } from '@nestjs/graphql';
|
||||
|
||||
import { FileUpload, GraphQLUpload } from 'graphql-upload';
|
||||
|
||||
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
|
||||
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
|
||||
import { ExecuteServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/execute-serverless-function.input';
|
||||
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { ServerlessFunctionDto } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto';
|
||||
import { ServerlessFunctionExecutionResultDTO } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result-d-t.o';
|
||||
import { serverlessFunctionGraphQLApiExceptionHandler } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-graphql-api-exception-handler.utils';
|
||||
|
||||
@UseGuards(JwtAuthGuard)
|
||||
@Resolver()
|
||||
export class ServerlessFunctionResolver {
|
||||
constructor(
|
||||
private readonly serverlessFunctionService: ServerlessFunctionService,
|
||||
) {}
|
||||
|
||||
@Mutation(() => ServerlessFunctionDto)
|
||||
async createOneServerlessFunction(
|
||||
@Args({ name: 'file', type: () => GraphQLUpload })
|
||||
file: FileUpload,
|
||||
@Args('name', { type: () => String }) name: string,
|
||||
@AuthWorkspace() { id: workspaceId }: Workspace,
|
||||
) {
|
||||
try {
|
||||
return await this.serverlessFunctionService.createOne(
|
||||
name,
|
||||
workspaceId,
|
||||
file,
|
||||
);
|
||||
} catch (error) {
|
||||
serverlessFunctionGraphQLApiExceptionHandler(error);
|
||||
}
|
||||
}
|
||||
|
||||
@Mutation(() => ServerlessFunctionExecutionResultDTO)
|
||||
async executeOneServerlessFunction(
|
||||
@Args() executeServerlessFunctionInput: ExecuteServerlessFunctionInput,
|
||||
@AuthWorkspace() { id: workspaceId }: Workspace,
|
||||
) {
|
||||
try {
|
||||
const { name, payload } = executeServerlessFunctionInput;
|
||||
|
||||
return {
|
||||
result: await this.serverlessFunctionService.executeOne(
|
||||
name,
|
||||
workspaceId,
|
||||
payload,
|
||||
),
|
||||
};
|
||||
} catch (error) {
|
||||
serverlessFunctionGraphQLApiExceptionHandler(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { join } from 'path';
|
||||
|
||||
import { FileUpload } from 'graphql-upload';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
|
||||
|
||||
import { ServerlessService } from 'src/engine/integrations/serverless/serverless.service';
|
||||
import {
|
||||
ServerlessFunctionEntity,
|
||||
ServerlessFunctionSyncStatus,
|
||||
} from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
|
||||
import {
|
||||
ServerlessFunctionException,
|
||||
ServerlessFunctionExceptionCode,
|
||||
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
|
||||
import { readFileContent } from 'src/engine/integrations/file-storage/utils/read-file-content';
|
||||
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
|
||||
import { SOURCE_FILE_NAME } from 'src/engine/integrations/serverless/drivers/constants/source-file-name';
|
||||
import { serverlessFunctionCreateHash } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-create-hash.utils';
|
||||
|
||||
@Injectable()
|
||||
export class ServerlessFunctionService {
|
||||
constructor(
|
||||
private readonly fileStorageService: FileStorageService,
|
||||
private readonly serverlessService: ServerlessService,
|
||||
@InjectRepository(ServerlessFunctionEntity, 'metadata')
|
||||
private readonly serverlessFunctionRepository: Repository<ServerlessFunctionEntity>,
|
||||
) {}
|
||||
|
||||
async executeOne(
|
||||
name: string,
|
||||
workspaceId: string,
|
||||
payload: object | undefined = undefined,
|
||||
) {
|
||||
const functionToExecute = await this.serverlessFunctionRepository.findOne({
|
||||
where: {
|
||||
name,
|
||||
workspaceId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!functionToExecute) {
|
||||
throw new ServerlessFunctionException(
|
||||
`Function does not exist`,
|
||||
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
functionToExecute.syncStatus === ServerlessFunctionSyncStatus.NOT_READY
|
||||
) {
|
||||
throw new ServerlessFunctionException(
|
||||
`Function is not ready to be executed`,
|
||||
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
return this.serverlessService.execute(functionToExecute, payload);
|
||||
}
|
||||
|
||||
async createOne(
|
||||
name: string,
|
||||
workspaceId: string,
|
||||
{ createReadStream, mimetype }: FileUpload,
|
||||
) {
|
||||
const existingServerlessFunction =
|
||||
await this.serverlessFunctionRepository.findOne({
|
||||
where: { name, workspaceId },
|
||||
});
|
||||
|
||||
if (existingServerlessFunction) {
|
||||
throw new ServerlessFunctionException(
|
||||
`Function already exists`,
|
||||
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_ALREADY_EXIST,
|
||||
);
|
||||
}
|
||||
|
||||
const typescriptCode = await readFileContent(createReadStream());
|
||||
|
||||
const serverlessFunction = await this.serverlessFunctionRepository.save({
|
||||
name,
|
||||
workspaceId,
|
||||
sourceCodeHash: serverlessFunctionCreateHash(typescriptCode),
|
||||
});
|
||||
|
||||
const fileFolder = join(
|
||||
FileFolder.ServerlessFunction,
|
||||
workspaceId,
|
||||
serverlessFunction.id,
|
||||
);
|
||||
|
||||
await this.fileStorageService.write({
|
||||
file: typescriptCode,
|
||||
name: SOURCE_FILE_NAME,
|
||||
mimeType: mimetype,
|
||||
folder: fileFolder,
|
||||
});
|
||||
|
||||
await this.serverlessService.build(serverlessFunction);
|
||||
await this.serverlessFunctionRepository.update(serverlessFunction.id, {
|
||||
syncStatus: ServerlessFunctionSyncStatus.READY,
|
||||
});
|
||||
|
||||
return await this.serverlessFunctionRepository.findOneByOrFail({
|
||||
id: serverlessFunction.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
export const serverlessFunctionCreateHash = (fileContent: string) => {
|
||||
return createHash('sha512')
|
||||
.update(fileContent)
|
||||
.digest('hex')
|
||||
.substring(0, 32);
|
||||
};
|
||||
@@ -0,0 +1,26 @@
|
||||
import {
|
||||
ServerlessFunctionException,
|
||||
ServerlessFunctionExceptionCode,
|
||||
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
|
||||
import {
|
||||
ConflictError,
|
||||
ForbiddenError,
|
||||
InternalServerError,
|
||||
NotFoundError,
|
||||
} from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
|
||||
|
||||
export const serverlessFunctionGraphQLApiExceptionHandler = (error: any) => {
|
||||
if (error instanceof ServerlessFunctionException) {
|
||||
switch (error.code) {
|
||||
case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND:
|
||||
throw new NotFoundError(error.message);
|
||||
case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_ALREADY_EXIST:
|
||||
throw new ConflictError(error.message);
|
||||
case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_READY:
|
||||
throw new ForbiddenError(error.message);
|
||||
default:
|
||||
throw new InternalServerError(error.message);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
};
|
||||
@@ -147,6 +147,16 @@ yarn command:prod cron:calendar:calendar-event-list-fetch
|
||||
['STORAGE_LOCAL_PATH', '.local-storage', 'data path (local storage)'],
|
||||
]}></ArticleTable>
|
||||
|
||||
### Custom Code Execution
|
||||
|
||||
<ArticleTable options={[
|
||||
['SERVERLESS_TYPE', 'local', "Serverless driver type: 'local' or 'lambda'"],
|
||||
['SERVERLESS_LAMBDA_REGION', '', 'Lambda Region'],
|
||||
['SERVERLESS_LAMBDA_ROLE', '', 'Lambda Role'],
|
||||
['SERVERLESS_LAMBDA_ACCESS_KEY_ID', '', 'Optional depending on the authentication method'],
|
||||
['SERVERLESS_LAMBDA_SECRET_ACCESS_KEY', '', 'Optional depending on the authentication method'],
|
||||
]}></ArticleTable>
|
||||
|
||||
### Message Queue
|
||||
|
||||
<ArticleTable options={[
|
||||
|
||||
Reference in New Issue
Block a user