From b1fbf4b683316a469fc8b52dc1753b863a8720ad Mon Sep 17 00:00:00 2001 From: BOHEUS <56270748+BOHEUS@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:07:10 +0000 Subject: [PATCH] E2E tests (#6717) Continuation of #6644 Now chromium browser is used in workspaces tests instead of firefox and screenshots after each test are properly saved in one folder when run from IDE and from terminal using `yarn test:e2e` command --- .github/workflows/playwright.yml.bak | 27 ++++++ .gitignore | 2 +- package.json | 1 + packages/twenty-e2e-testing/.env.example | 22 ++++- packages/twenty-e2e-testing/.gitignore | 15 ++-- packages/twenty-e2e-testing/README.md | 11 ++- .../config/customreporter.ts | 33 +++++++ .../drivers/shell_driver.ts | 13 +++ .../twenty-e2e-testing/e2e/companies.spec.ts | 14 --- .../lib/fixtures/screenshot.ts | 32 +++++++ .../twenty-e2e-testing/playwright.config.ts | 88 ++++++++++++++----- .../tests/companies.spec.ts | 19 ++++ .../twenty-e2e-testing/tests/login.setup.ts | 18 ++++ .../tests/workspaces.spec.ts | 66 ++++++++++++++ yarn.lock | 1 + 15 files changed, 316 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/playwright.yml.bak create mode 100644 packages/twenty-e2e-testing/config/customreporter.ts create mode 100644 packages/twenty-e2e-testing/drivers/shell_driver.ts delete mode 100644 packages/twenty-e2e-testing/e2e/companies.spec.ts create mode 100644 packages/twenty-e2e-testing/lib/fixtures/screenshot.ts create mode 100644 packages/twenty-e2e-testing/tests/companies.spec.ts create mode 100644 packages/twenty-e2e-testing/tests/login.setup.ts create mode 100644 packages/twenty-e2e-testing/tests/workspaces.spec.ts diff --git a/.github/workflows/playwright.yml.bak b/.github/workflows/playwright.yml.bak new file mode 100644 index 000000000..cffb50287 --- /dev/null +++ b/.github/workflows/playwright.yml.bak @@ -0,0 +1,27 @@ +name: Playwright Tests +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] +jobs: + test: + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: Install dependencies + run: npm install -g yarn && yarn + - name: Install Playwright Browsers + run: yarn playwright install --with-deps + - name: Run Playwright tests + run: yarn test:e2e companies + - uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 diff --git a/.gitignore b/.gitignore index 376216c45..febe678d4 100644 --- a/.gitignore +++ b/.gitignore @@ -26,5 +26,5 @@ dist storybook-static *.tsbuildinfo .eslintcache -.cache .nyc_output +test-results/ \ No newline at end of file diff --git a/package.json b/package.json index 33bc0b492..c4b1db569 100644 --- a/package.json +++ b/package.json @@ -222,6 +222,7 @@ "@nx/storybook": "18.3.3", "@nx/vite": "18.3.3", "@nx/web": "18.3.3", + "@playwright/test": "^1.46.0", "@sentry/types": "^7.109.0", "@storybook/addon-actions": "^7.6.3", "@storybook/addon-coverage": "^1.0.0", diff --git a/packages/twenty-e2e-testing/.env.example b/packages/twenty-e2e-testing/.env.example index 9ff92d019..29209f8c8 100644 --- a/packages/twenty-e2e-testing/.env.example +++ b/packages/twenty-e2e-testing/.env.example @@ -1,2 +1,22 @@ # Note that provide always without trailing forward slash to have expected behaviour -FRONTEND_BASE_URL="http://localhost:3001" +FRONTEND_BASE_URL=http://localhost:3001 +CI_DEFAULT_BASE_URL=https://demo.twenty.com +DEFAULT_LOGIN=tim@apple.dev +NEW_WORKSPACE_LOGIN=test@apple.dev +DEMO_DEFAULT_LOGIN=noah@demo.dev +DEFAULT_PASSWORD=Applecar2025 +WEBSITE_URL=https://twenty.com + +# === DO NOT USE, WORK IN PROGRESS === +# This URL must have trailing forward slash as all REST API endpoints have object after it +# Documentation for REST API: https://twenty.com/developers/rest-api/core#/ +# REST_API_BASE_URL=http://localhost:3000/rest/ + +# Documentation for GraphQL API: https://twenty.com/developers/graphql/core +# GRAPHQL_BASE_URL=http://localhost:3000/graphql + +# Without this key, all API tests will fail, to generate this key +# Log in to Twenty workspace, go to Settings > Developers, generate new key and paste it here +# In order to use it, header Authorization: Bearer token must be used +# This key works for both REST and GraphQL API +# API_DEV_KEY=fill_with_proper_key \ No newline at end of file diff --git a/packages/twenty-e2e-testing/.gitignore b/packages/twenty-e2e-testing/.gitignore index 68c5d18f0..676066d51 100644 --- a/packages/twenty-e2e-testing/.gitignore +++ b/packages/twenty-e2e-testing/.gitignore @@ -1,5 +1,10 @@ -node_modules/ -/test-results/ -/playwright-report/ -/blob-report/ -/playwright/.cache/ +test-results/ +playwright-report/ +blob-report/ +playwright/.cache/ +run_results/ +playwright-report/.last-run.json +results/ +run_results/.playwright-artifacts-0/ +run_results/.playwright-artifacts-1/ +.auth/ \ No newline at end of file diff --git a/packages/twenty-e2e-testing/README.md b/packages/twenty-e2e-testing/README.md index 222f1d807..4f39ca3c8 100644 --- a/packages/twenty-e2e-testing/README.md +++ b/packages/twenty-e2e-testing/README.md @@ -1,8 +1,8 @@ # Twenty e2e Testing -## Install +## Prerequisite -Don't forget to install the browsers before launching the tests : +Installing the browsers: ``` yarn playwright install @@ -35,3 +35,10 @@ yarn run test:e2e ``` yarn run test:e2e:debug ``` + +## Q&A + +#### Why there's `path.resolve()` everywhere? +That's thanks to differences in root directory when running tests using commands and using IDE. When running tests with commands, +the root directory is `twenty/packages/twenty-e2e-testing`, for IDE it depends on how someone sets the configuration. This way, it +ensures that no matter which IDE or OS Shell is used, the result will be the same. diff --git a/packages/twenty-e2e-testing/config/customreporter.ts b/packages/twenty-e2e-testing/config/customreporter.ts new file mode 100644 index 000000000..62a602ef8 --- /dev/null +++ b/packages/twenty-e2e-testing/config/customreporter.ts @@ -0,0 +1,33 @@ +import { + Reporter, + FullConfig, + Suite, + TestCase, + TestResult, + FullResult, +} from '@playwright/test/reporter'; + +class CustomReporter implements Reporter { + constructor(options: { customOption?: string } = {}) { + console.log( + `my-awesome-reporter setup with customOption set to ${options.customOption}`, + ); + } + + onBegin(config: FullConfig, suite: Suite) { + console.log(`Starting the run with ${suite.allTests().length} tests`); + } + + onTestBegin(test: TestCase) { + console.log(`Starting test ${test.title}`); + } + + onTestEnd(test: TestCase, result: TestResult) { + console.log(`Finished test ${test.title}: ${result.status}`); + } + + onEnd(result: FullResult) { + console.log(`Finished the run: ${result.status}`); + } +} +export default CustomReporter; diff --git a/packages/twenty-e2e-testing/drivers/shell_driver.ts b/packages/twenty-e2e-testing/drivers/shell_driver.ts new file mode 100644 index 000000000..cf293c032 --- /dev/null +++ b/packages/twenty-e2e-testing/drivers/shell_driver.ts @@ -0,0 +1,13 @@ +import { exec } from 'child_process'; + +export async function sh(cmd) { + return new Promise((resolve, reject) => { + exec(cmd, (err, stdout, stderr) => { + if (err) { + reject(err); + } else { + resolve({ stdout, stderr }); + } + }); + }); +} diff --git a/packages/twenty-e2e-testing/e2e/companies.spec.ts b/packages/twenty-e2e-testing/e2e/companies.spec.ts deleted file mode 100644 index 48485da04..000000000 --- a/packages/twenty-e2e-testing/e2e/companies.spec.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { expect, test } from '@playwright/test'; - -test.describe('visible table', () => { - test('table should be visible on navigation to /objects/companies', async ({ - page, - }) => { - // Navigate to the page - await page.goto('/objects/companies'); - - // Check if the table is visible - const table = page.locator('table'); - await expect(table).toBeVisible(); - }); -}); diff --git a/packages/twenty-e2e-testing/lib/fixtures/screenshot.ts b/packages/twenty-e2e-testing/lib/fixtures/screenshot.ts new file mode 100644 index 000000000..abea94f61 --- /dev/null +++ b/packages/twenty-e2e-testing/lib/fixtures/screenshot.ts @@ -0,0 +1,32 @@ +import { test as base } from '@playwright/test'; +import path from 'path'; + +const date = new Date(); + +export const test = base.extend<{ screenshotHook: void }>({ + screenshotHook: [ + async ({ page, browserName }, use, workerInfo) => { + // here everything is the same as beforeEach() + // goto is to go to website as login setup saves the cookies of logged-in user, not the state of browser + await page.goto('/'); + await use(); // here is the code of test + // here everything is the same as afterEach() + // automatic fixture of making screenshot after each test + await page.screenshot({ + path: path.resolve( + __dirname, + '..', + '..', + 'results', + 'screenshots', + `${workerInfo.project.name}`, + browserName, + `${date.toISOString()}.png`, + ), + }); + }, + { auto: true }, // automatic fixture runs with every test + ], +}); + +export { expect } from '@playwright/test'; diff --git a/packages/twenty-e2e-testing/playwright.config.ts b/packages/twenty-e2e-testing/playwright.config.ts index 4b4f081de..964b31e32 100644 --- a/packages/twenty-e2e-testing/playwright.config.ts +++ b/packages/twenty-e2e-testing/playwright.config.ts @@ -1,43 +1,85 @@ import { defineConfig, devices } from '@playwright/test'; - import { config } from 'dotenv'; +import path from 'path'; + config(); +/* === Run your local dev server before starting the tests === */ + /** * See https://playwright.dev/docs/test-configuration. - * See https://playwright.dev/docs/trace-viewer to Collect trace when retrying the failed test */ export default defineConfig({ - testDir: 'e2e', - /* Run tests in files in parallel */ - fullyParallel: true, - reporter: 'html', - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + testDir: '.', + outputDir: 'run_results/', // directory for screenshots and videos + snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}', // just in case, do not delete it + fullyParallel: true, // false only for specific tests, overwritten in specific projects or global setups of projects + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, // undefined = amount of projects * amount of tests + timeout: 30 * 1000, // timeout can be changed use: { - /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: process.env.FRONTEND_BASE_URL ?? 'http://localhost:3001', + baseURL: process.env.CI + ? process.env.CI_DEFAULT_BASE_URL + : (process.env.FRONTEND_BASE_URL ?? 'http://localhost:3001'), + trace: 'retain-on-failure', // trace takes EVERYTHING from page source, records every single step, should be used only when normal debugging won't work + screenshot: 'on', // either 'on' here or in different method in modules, if 'on' all screenshots are overwritten each time the test is run + headless: true, // instead of changing it to false, run 'yarn test:e2e:debug' or 'yarn test:e2e:ui' + testIdAttribute: 'data-testid', // taken from Twenty source + viewport: { width: 1920, height: 1080 }, // most laptops use this resolution + launchOptions: { + slowMo: 500, // time in milliseconds between each step, better to use it than explicitly define timeout in tests + }, }, - - /* Configure projects for major browsers */ + expect: { + timeout: 5000, + }, + reporter: [['html', { open: 'never' }]], projects: [ { - name: 'chromium', - use: { ...devices['Desktop Chrome'] }, + name: 'Login setup', + testMatch: /login\.setup\.ts/, // finds all tests matching this regex, in this case only 1 test should be found + }, + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + storageState: path.resolve(__dirname, '.auth', 'user.json'), // takes saved cookies from directory + }, + dependencies: ['Login setup'], // forces to run login setup before running tests from this project - CASE SENSITIVE }, - { name: 'firefox', - use: { ...devices['Desktop Firefox'] }, + use: { + ...devices['Desktop Firefox'], + storageState: path.resolve(__dirname, '.auth', 'user.json'), + }, + dependencies: ['Login setup'], }, - { - name: 'webkit', - use: { ...devices['Desktop Safari'] }, - }, + //{ + // name: 'webkit', + // use: { ...devices['Desktop Safari'] }, + //}, - { - name: 'Google Chrome', - use: { ...devices['Desktop Chrome'], channel: 'chrome' }, - }, + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + //{ + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + //}, + //{ + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + //}, ], }); diff --git a/packages/twenty-e2e-testing/tests/companies.spec.ts b/packages/twenty-e2e-testing/tests/companies.spec.ts new file mode 100644 index 000000000..b8f78c7ec --- /dev/null +++ b/packages/twenty-e2e-testing/tests/companies.spec.ts @@ -0,0 +1,19 @@ +import { test, expect } from '../lib/fixtures/screenshot'; +import { config } from 'dotenv'; +import path = require('path'); +config({ path: path.resolve(__dirname, '..', '.env') }); + +test.describe('Basic check', () => { + test('Checking if table in Companies is visible', async ({ page }) => { + await expect(page.getByTestId('tooltip').nth(0)).toHaveText('Companies'); + await expect(page.getByTestId('tooltip').nth(0)).toBeVisible(); + expect(page.url()).toContain('/companies'); + await expect(page.locator('table')).toBeVisible(); + await expect(page.locator('tbody > tr')).toHaveCount(13); // shouldn't be hardcoded in case of tests on demo + }); + + test('', async ({ page }) => { + await page.getByRole('link', { name: 'Opportunities' }).click(); + await expect(page.locator('table')).toBeVisible(); + }); +}); diff --git a/packages/twenty-e2e-testing/tests/login.setup.ts b/packages/twenty-e2e-testing/tests/login.setup.ts new file mode 100644 index 000000000..defd19685 --- /dev/null +++ b/packages/twenty-e2e-testing/tests/login.setup.ts @@ -0,0 +1,18 @@ +import { test as setup, expect } from '@playwright/test'; +import path from 'path'; + +setup('Login test', async ({ page }) => { + await page.goto('/'); + await page.getByRole('button', { name: 'Continue With Email' }).click(); + await page.getByPlaceholder('Email').fill(process.env.DEFAULT_LOGIN); + await page.getByRole('button', { name: 'Continue', exact: true }).click(); + await page.getByPlaceholder('Password').fill(process.env.DEFAULT_PASSWORD); + await page.getByRole('button', { name: 'Sign in' }).click(); + await expect(page.getByText('Welcome to Twenty')).not.toBeVisible(); + + // End of authentication steps. + + await page.context().storageState({ + path: path.resolve(__dirname, '..', '.auth', 'user.json'), + }); +}); diff --git a/packages/twenty-e2e-testing/tests/workspaces.spec.ts b/packages/twenty-e2e-testing/tests/workspaces.spec.ts new file mode 100644 index 000000000..f8cff58f5 --- /dev/null +++ b/packages/twenty-e2e-testing/tests/workspaces.spec.ts @@ -0,0 +1,66 @@ +import { test, expect } from '@playwright/test'; +import { sh } from '../drivers/shell_driver'; + +test.describe('', () => { + test('Testing logging', async ({ page }) => { + await page.goto('/'); + await page.getByRole('button', { name: 'Continue With Email' }).click(); + await page.getByPlaceholder('Email').fill('tim@apple.dev'); + await page.getByRole('button', { name: 'Continue', exact: true }).click(); + await page.getByPlaceholder('Password').fill('Applecar2025'); + await page.getByRole('button', { name: 'Sign in' }).click(); + await expect(page.getByText('Welcome to Twenty')).not.toBeVisible(); + expect(page.url()).not.toContain('/welcome'); + await page.getByRole('link', { name: 'Opportunities' }).click(); + await expect(page.locator('tbody > tr')).toHaveCount(4); + }); + + test('Creating new workspace', async ({ page, browserName }) => { + // this test must use only 1 browser, otherwise it will lead to success and fail (1 workspace is created instead of x workspaces) + if (browserName == 'chromium') { + await page.goto('/'); + await page.getByRole('button', { name: 'Continue With Email' }).click(); + await page.getByPlaceholder('Email').fill('test@apple.dev'); // email must be changed each time test is run + await page.getByPlaceholder('Email').press('Enter'); // otherwise if tests fails after this step, new workspace is created + await page.getByPlaceholder('Password').fill('Applecar2025'); + await page.getByPlaceholder('Password').press('Enter'); + await page.getByPlaceholder('Apple').fill('Test'); + await page.getByRole('button', { name: 'Continue' }).click(); + await page.getByPlaceholder('Tim').click(); + await page.getByPlaceholder('Tim').fill('Test2'); + await page.getByPlaceholder('Cook').click(); + await page.getByPlaceholder('Cook').fill('Test2'); + await page.getByRole('button', { name: 'Continue' }).click(); + await page.getByText('Continue without sync').click(); + await page.getByRole('button', { name: 'Finish' }).click(); + await expect(page.locator('table')).toBeVisible({ timeout: 1000 }); + } + }); + + test('Syncing all workspaces', async () => { + await sh('npx nx run twenty-server:command workspace:sync-metadata -f'); + await sh('npx nx run twenty-server:command workspace:sync-metadata -f'); + }); + + test('Resetting database', async ({ page, browserName }) => { + if (browserName === 'chromium') { + await sh('yarn nx database:reset twenty-server'); // if this command fails for any reason, database must be restarted manually using the same command because database is in unstable state + await page.goto('/'); + await page.getByRole('button', { name: 'Continue With Email' }).click(); + await page.getByPlaceholder('Email').fill('tim@apple.dev'); + await page.getByRole('button', { name: 'Continue' }).click(); + await page.getByPlaceholder('Password').fill('Applecar2025'); + await page.getByRole('button', { name: 'Sign in' }).click(); + await page.getByRole('link', { name: 'Companies' }).click(); + expect(page.url()).toContain('/companies'); + await expect(page.locator('table')).toBeVisible(); + } + }); + + test('Seeding database', async ({ page, browserName }) => { + if (browserName === 'chromium') { + await sh('npx nx workspace:seed:demo'); + await page.goto('/'); + } + }); +}); diff --git a/yarn.lock b/yarn.lock index d6163be0f..e31171346 100644 --- a/yarn.lock +++ b/yarn.lock @@ -47214,6 +47214,7 @@ __metadata: "@nx/vite": "npm:18.3.3" "@nx/web": "npm:18.3.3" "@octokit/graphql": "npm:^7.0.2" + "@playwright/test": "npm:^1.46.0" "@ptc-org/nestjs-query-core": "npm:^4.2.0" "@ptc-org/nestjs-query-typeorm": "npm:4.2.1-alpha.2" "@react-email/components": "npm:0.0.12"