Begin moving to postgres spilo + adding pgvector (#8309)

We will remove the `twenty-postgres` image that was used for local
development and only use `twenty-postgres-pilo` (which we use in prod),
bringing the development environment closer to prod and avoiding having
to maintain 2 images.


Instead of provisioning the super user after the db initialization, we
directly rely on the superuser provided by Spilo for simplicity. We also
introduce a change that tries to create the right database (`default` or
`test`) based on the context.
  

How to test:
```
docker build -t twentycrm/twenty-postgres-spilo:latest -f ./packages/twenty-docker/twenty-postgres-spilo/Dockerfile .
docker images --no-trunc | grep twenty-postgres-spilo
postgres-on-docker:
	docker run \
	--name twenty_pg \
	-e PGUSER_SUPERUSER=twenty \
	-e PGPASSWORD_SUPERUSER=twenty \
	-e ALLOW_NOSSL=true \
	-v twenty_db_data:/home/postgres/pgdata \
	-p 5432:5432 \
	REPLACE_WITH_IMAGE_ID
```

.github/workflows/ci-server.yaml

@@ -18,12 +18,19 @@ jobs:
       NX_REJECT_UNKNOWN_LOCAL_CACHE: 0
     services:
       postgres:
-        image: twentycrm/twenty-postgres
+        image: twentycrm/twenty-postgres-spilo
         env:
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_USER: postgres
+          PGUSER_SUPERUSER: postgres
+          PGPASSWORD_SUPERUSER: twenty
+          ALLOW_NOSSL: "true"
+          SPILO_PROVIDER: "local"
         ports:
           - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
       redis:
         image: redis
         ports:
@@ -63,6 +70,11 @@ jobs:
       - name: Server / Write .env
         if: steps.changed-files.outputs.any_changed == 'true'
         run: npx nx reset:env twenty-server
+      - name: Server / Create DB
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          PGPASSWORD=twenty psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
+          PGPASSWORD=twenty psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
       - name: Worker / Run
         if: steps.changed-files.outputs.any_changed == 'true'
         run: npx nx run twenty-server:worker:ci
@@ -109,12 +121,19 @@ jobs:
     needs: server-setup
     services:
       postgres:
-        image: twentycrm/twenty-postgres
+        image: twentycrm/twenty-postgres-spilo
         env:
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_USER: postgres
+          PGUSER_SUPERUSER: postgres
+          PGPASSWORD_SUPERUSER: twenty
+          ALLOW_NOSSL: "true"
+          SPILO_PROVIDER: "local"
         ports:
           - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
       redis:
         image: redis
         ports:

.github/workflows/ci-test-docker-compose.yaml

@@ -8,7 +8,7 @@ concurrency:
 
 jobs:
   test:
-    timeout-minutes: 10
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
@@ -36,20 +36,42 @@ jobs:
 
         yq eval 'del(.services.db.image)' -i docker-compose.yml
         yq eval '.services.db.build.context = "../../"' -i docker-compose.yml
-        yq eval '.services.db.build.dockerfile = "./packages/twenty-docker/twenty-postgres/Dockerfile"' -i docker-compose.yml
+        yq eval '.services.db.build.dockerfile = "./packages/twenty-docker/twenty-postgres-spilo/Dockerfile"' -i docker-compose.yml
 
         echo "Setting up .env file..."
         cp .env.example .env
         echo "Generating secrets..."
         echo "# === Randomly generated secrets ===" >>.env
         echo "APP_SECRET=$(openssl rand -base64 32)" >>.env
-        echo "POSTGRES_ADMIN_PASSWORD=$(openssl rand -base64 32)" >>.env
+        echo "PGPASSWORD_SUPERUSER=$(openssl rand -base64 32)" >>.env
 
-        echo "Starting server..."
-        docker compose up -d
+        echo "Docker compose up..."
+        docker compose up -d || {
+            echo "Docker compose failed to start"
+            docker compose logs
+            exit 1
+        }
         docker compose logs db server -f &
         pid=$!
 
+        echo "Waiting for database to start..."
+        count=0
+        while [ ! $(docker inspect --format='{{.State.Health.Status}}' twenty-db-1) = "healthy" ]; do
+            sleep 1;
+            count=$((count+1));
+            if [ $(docker inspect --format='{{.State.Status}}' twenty-db-1) = "exited" ]; then
+                echo "Database exited"
+                docker compose logs db
+                exit 1
+            fi
+            if [ $count -gt 300 ]; then
+                echo "Failed to start database after 5 minutes"
+                docker compose logs db
+                exit 1
+            fi
+            echo "Still waiting for database... (${count}/60)"
+        done
+
         echo "Waiting for server to start..."
         count=0
         while [ ! $(docker inspect --format='{{.State.Health.Status}}' twenty-server-1) = "healthy" ]; do
@@ -57,11 +79,14 @@ jobs:
             count=$((count+1));
             if [ $(docker inspect --format='{{.State.Status}}' twenty-server-1) = "exited" ]; then
                 echo "Server exited"
+                docker compose logs server
                 exit 1
             fi
             if [ $count -gt 300 ]; then
-                echo "Failed to start server"
+                echo "Failed to start server after 5 minutes"
+                docker compose logs server
                 exit 1
             fi
+            echo "Still waiting for server... (${count}/300s)"
         done
       working-directory: ./packages/twenty-docker/

.github/workflows/ci-website.yaml

@@ -17,12 +17,19 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: twentycrm/twenty-postgres
+        image: twentycrm/twenty-postgres-spilo
         env:
-          POSTGRES_PASSWORD: twenty
-          POSTGRES_USER: twenty
+          PGUSER_SUPERUSER: postgres
+          PGPASSWORD_SUPERUSER: twenty
+          ALLOW_NOSSL: "true"
+          SPILO_PROVIDER: "local"
         ports:
           - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v4
         with:
@@ -37,16 +44,20 @@ jobs:
         if: steps.changed-files.outputs.changed == 'true' 
         uses: ./.github/workflows/actions/yarn-install
 
+      - name: Server / Create DB
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: PGPASSWORD=twenty psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
+
       - name: Website / Run migrations
         if: steps.changed-files.outputs.changed == 'true' 
         run: npx nx database:migrate twenty-website 
         env:
-          DATABASE_PG_URL: postgres://twenty:twenty@localhost:5432/default
+          DATABASE_PG_URL: postgres://postgres:twenty@localhost:5432/default
       - name: Website / Build Website
         if: steps.changed-files.outputs.changed == 'true' 
         run: npx nx build twenty-website
         env:
-          DATABASE_PG_URL: postgres://twenty:twenty@localhost:5432/default
+          DATABASE_PG_URL: postgres://postgres:twenty@localhost:5432/default
 
       - name: Mark as VALID
         if: steps.changed-files.outputs.changed != 'true'  # If no changes, mark as valid