chore: use Nx affected tasks in CI (#5110)

Closes #5097

- Uses "nx affected" to detect what projects need to be checked in the
current PR (for now, `ci-front` and `ci-server` workflows only).
- Caches results of certain tasks (`lint`, `typecheck`, `test`,
`storybook:build`) when a PR pipeline runs. The next runs of the same
PR's pipeline will then be able to reuse the PR's task cache to execute
tasks faster.
- Caches Yarn's cache folder to install dependencies faster in CI jobs.
- Rewrites the node modules cache/install steps as a custom, reusable
Github action.
- Distributes `ci-front` jobs with a "matrix" strategy.
- Sets common tasks config at the root `nx.json`. For instance, to
activate the `typecheck` task in a project, add `typecheck: {}` to its
`project.json` and it'll use the default config set in `nx.json` for the
`typecheck` task. Options can be overridden in each individual
`project.json` if needed.
- Adds "scope" tags to some projects: `scope:frontend`, `scope:backend`,
`scope:shared`. An eslint rule ensures that `scope:frontend` only
depends on `scope:frontent` or `scope:shared` projects, same for
`scope:backend`. These tags are used by `nx affected` to filter projects
by scope and generates different task cache keys according to the
requested scope.
- Enables checks for twenty-emails in the `ci-server` workflow.

.github/workflows/actions/nx-affected/action.yaml

@@ -0,0 +1,21 @@
+name: Nx Affected CI
+inputs:
+  parallel:
+    required: false
+    types: [number]
+    default: 3
+  tag:
+    required: false
+    types: [string]
+  tasks:
+    required: true
+    types: [string]
+
+runs:
+  using: "composite"
+  steps:
+    - name: Get last successful commit
+      uses: nrwl/nx-set-shas@v4
+    - name: Run affected command
+      shell: bash
+      run: npx nx affected --nxBail --configuration=ci -t=${{ inputs.tasks }} --parallel=${{ inputs.parallel }} --exclude='*,!tag:${{ inputs.tag }}'

.github/workflows/actions/task-cache/action.yaml

@@ -0,0 +1,29 @@
+name: Restore Tasks Cache CI
+inputs:
+  tag:
+    required: false
+    types: [string]
+  tasks:
+    required: false
+    types: [string]
+    default: all
+  suffix:
+    required: false
+    types: [string]
+
+runs:
+  using: "composite"
+  steps:
+    - name: Compute tasks key
+      id: tasks-key
+      shell: bash
+      run: echo "key=${{ inputs.tasks }}" | tr , - >> $GITHUB_OUTPUT
+    - name: Restore tasks cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          .cache
+          .nx/cache
+        key: tasks-cache-${{ inputs.tag }}-${{ steps.tasks-key.outputs.key }}${{ inputs.suffix }}-${{ github.sha }}
+        restore-keys: |
+          tasks-cache-${{ inputs.tag }}-${{ steps.tasks-key.outputs.key }}${{ inputs.suffix }}-

.github/workflows/actions/yarn-install/action.yaml

@@ -0,0 +1,23 @@
+name: Yarn Install
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Node.js and get yarn cache
+      uses: actions/setup-node@v3
+      with:
+        node-version: "18"
+        cache: yarn
+    - name: Cache node modules
+      id: node-modules-cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          node_modules
+          packages/*/node_modules
+        key: root-node_modules-${{ hashFiles('yarn.lock') }}
+        restore-keys: root-node_modules-
+    - name: Install Dependencies
+      shell: bash
+      run: yarn --immutable --check-cache
+      if: steps.node-modules-cache.outputs.cache-hit != 'true'

.github/workflows/ci-chromatic.yaml

@@ -27,17 +27,13 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Front / Write .env
         run: |
           cd packages/twenty-front
           touch .env
           echo "REACT_APP_SERVER_BASE_URL: $REACT_APP_SERVER_BASE_URL" >> .env
-      - name: Front / Install Dependencies
-        run: yarn
       - name: Publish to Chromatic
         run: |
           cd packages/twenty-front

.github/workflows/ci-chrome-extension.yaml

@@ -15,33 +15,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  chrome-extension-yarn-install:
-    runs-on: ci-8-cores
-    env:
-      VITE_SERVER_BASE_URL: http://localhost:3000
-      VITE_FRONT_BASE_URL: http://localhost:3001
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Cache chrome extension node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-chrome-extension/node_modules
-          key: chrome-extension-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: chrome-extension-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: Chrome Extension / Install Dependencies
-        run: yarn
   chrome-extension-build:
-    needs: chrome-extension-yarn-install
     runs-on: ubuntu-latest
     env:
       VITE_SERVER_BASE_URL: http://localhost:3000
@@ -52,21 +26,7 @@ jobs:
         with:
             access_token: ${{ github.token }}
       - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Cache chrome extension node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-chrome-extension/node_modules
-          key: chrome-extension-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: chrome-extension-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Chrome Extension / Run build
         run: npx nx build twenty-chrome-extension

.github/workflows/ci-docs.yaml

@@ -21,12 +21,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Docs / Install Dependencies
-        run: yarn
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Docs / Build Documentation
         run: npx nx build twenty-docs
   vale:

.github/workflows/ci-front.yaml

@@ -17,167 +17,63 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  front-yarn-install:
-    runs-on: ci-8-cores
-    env:
-      REACT_APP_SERVER_BASE_URL: http://localhost:3000
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Cache front node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: Front / Install Dependencies
-        run: yarn
-  front-pages-sb-test:
-    needs: front-yarn-install
+  front-sb-test:
     runs-on: ci-8-cores
+    strategy:
+      matrix:
+        storybook_scope: [pages, modules]
     env:
       REACT_APP_SERVER_BASE_URL: http://localhost:3000
+      STORYBOOK_SCOPE: ${{ matrix.storybook_scope }}
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.11.0
         with:
-            access_token: ${{ github.token }}
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Front / Write .env
-        run: |
-          cd packages/twenty-front
-          cp .env.example .env
-      - name: Cache front node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
+          access_token: ${{ github.token }}
+      - name: Fetch local actions
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Install Playwright
         run: cd packages/twenty-front && npx playwright install
+      - name: Front / Restore Storybook Task Cache
+        uses: ./.github/workflows/actions/task-cache
+        with:
+          tag: scope:frontend
+          tasks: storybook:build
+          suffix: _${{ matrix.storybook_scope }}
+      - name: Front / Write .env
+        run: npx nx reset:env twenty-front
       - name: Run storybook tests
         run: |
           npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
-            "STORYBOOK_SCOPE=pages npx nx run twenty-front:storybook:static:ci" \
-            "npx wait-on tcp:6006 && STORYBOOK_SCOPE=pages npx nx run twenty-front:storybook:test"
-  front-modules-sb-test:
-    needs: front-yarn-install
-    runs-on: ci-4-cores
-    env:
-      REACT_APP_SERVER_BASE_URL: http://localhost:3000
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-            access_token: ${{ github.token }}
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Front / Write .env
-        run: |
-          cd packages/twenty-front
-          cp .env.example .env
-      - name: Cache front node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: Install Playwright
-        run: cd packages/twenty-front && npx playwright install
-      - name: Run storybook tests
-        run: |
-          npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
-            "STORYBOOK_SCOPE=modules npx nx run twenty-front:storybook:static:ci" \
-            "npx wait-on tcp:6006 && STORYBOOK_SCOPE=modules npx nx run twenty-front:storybook:test"
-  front-lint-tsc:
-    needs: front-yarn-install
+            "npx nx storybook:static twenty-front" \
+            "npx wait-on tcp:6006 && npx nx storybook:test twenty-front"
+  front-task:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        task: [lint, typecheck, test]
     env:
       REACT_APP_SERVER_BASE_URL: http://localhost:3000
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.11.0
         with:
-            access_token: ${{ github.token }}
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
+          access_token: ${{ github.token }}
+      - name: Fetch local actions and base branch history
+        uses: actions/checkout@v4
         with:
-          node-version: "18"
-      - name: Cache front node modules
-        uses: actions/cache@v3
+          fetch-depth: 0
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
+      - name: Front / Restore Task Cache
+        uses: ./.github/workflows/actions/task-cache
         with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
+          tag: scope:frontend
+          tasks: ${{ matrix.task }}
+      - name: Front / Run task
+        uses: ./.github/workflows/actions/nx-affected
         with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: UI / Run linter
-        run: npx nx lint twenty-ui
-      - name: UI / Run Typescript Check
-        run: npx nx run twenty-ui:typecheck:ci
-      - name: Front / Run linter
-        run: npx nx run twenty-front:lint:ci
-      - name: Front / Run Typescript Check
-        run: npx nx run twenty-front:typecheck:ci
-  front-jest:
-    needs: front-yarn-install
-    runs-on: ubuntu-latest
-    env:
-      REACT_APP_SERVER_BASE_URL: http://localhost:3000
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-            access_token: ${{ github.token }}
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Cache front node modules
-        uses: actions/cache@v3
-        with:
-          path: packages/twenty-front/node_modules
-          key: front-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: front-node_modules-
-      - name: Cache root node modules
-        uses: actions/cache@v3
-        with:
-          path: node_modules
-          key: root-node_modules-${{hashFiles('yarn.lock')}}
-          restore-keys: root-node_modules-
-      - name: Front / Run jest
-        run: npx nx test twenty-front --configuration=ci
+          tag: scope:frontend
+          tasks: ${{ matrix.task }}

.github/workflows/ci-server.yaml

@@ -28,22 +28,29 @@ jobs:
         ports:
           - 5432:5432
     steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
+      - name: Fetch local actions and base branch history
+        uses: actions/checkout@v4
         with:
-          node-version: "18"
-      - name: Server / Install Dependencies
-        run: yarn
-      - name: Server / Run linter
-        run: npx nx lint twenty-server
-      - name: Server / Run jest tests
-        run: npx nx test:unit twenty-server
+          fetch-depth: 0
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
+      - name: Server / Restore Tasks Cache
+        uses: ./.github/workflows/actions/task-cache
+        with:
+          tag: scope:backend
+      - name: Server / Run lint & typecheck
+        uses: ./.github/workflows/actions/nx-affected
+        with:
+          tag: scope:backend
+          tasks: lint,typecheck
+      - name: Server / Run tests
+        uses: ./.github/workflows/actions/nx-affected
+        with:
+          tag: scope:backend
+          tasks: test
       - name: Server / Build
         run: npx nx build twenty-server
       - name: Server / Write .env
-        run: |
-          cd packages/twenty-server
-          cp .env.example .env
+        run: npx nx reset:env twenty-server
       - name: Worker / Run
         run: MESSAGE_QUEUE_TYPE=sync npx nx worker twenty-server

.github/workflows/ci-utils.yaml

@@ -21,12 +21,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Utils / Install Dependencies
-        run: yarn
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Utils / Run Danger.js
         run: cd packages/twenty-utils && npx nx danger:ci
         env:

.github/workflows/ci-website.yaml

@@ -27,12 +27,8 @@ jobs:
           - 5432:5432
     steps:
       - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      - name: Website / Install Dependencies
-        run: yarn
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
       - name: Website / Run migrations
         run: npx nx database:migrate twenty-website 
         env: