diff --git a/Makefile b/Makefile index fa9a04ef..8cc1b01e 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,6 @@ build-deps: build: build-deps make -C packages/apps/http-cache image - make -C packages/apps/postgres image make -C packages/apps/mysql image make -C packages/apps/clickhouse image make -C packages/apps/kubernetes image diff --git a/packages/apps/postgres/Chart.yaml b/packages/apps/postgres/Chart.yaml index 1183562f..e353d6a8 100644 --- a/packages/apps/postgres/Chart.yaml +++ b/packages/apps/postgres/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.14.0 +version: 0.15.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/packages/apps/postgres/Makefile b/packages/apps/postgres/Makefile index c0a894c0..264adfcf 100644 --- a/packages/apps/postgres/Makefile +++ b/packages/apps/postgres/Makefile @@ -1,24 +1,4 @@ -POSTGRES_BACKUP_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml) - -include ../../../scripts/common-envs.mk include ../../../scripts/package.mk generate: readme-generator -v values.yaml -s values.schema.json -r README.md - -image: - docker buildx build images/postgres-backup \ - --provenance false \ - --builder=$(BUILDER) \ - --platform=$(PLATFORM) \ - --tag $(REGISTRY)/postgres-backup:$(call settag,$(POSTGRES_BACKUP_TAG)) \ - --cache-from type=registry,ref=$(REGISTRY)/postgres-backup:latest \ - --cache-to type=inline \ - --metadata-file images/postgres-backup.json \ - --push=$(PUSH) \ - --label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \ - --load=$(LOAD) - echo "$(REGISTRY)/postgres-backup:$(call settag,$(POSTGRES_BACKUP_TAG))@$$(yq e '."containerimage.digest"' images/postgres-backup.json -o json -r)" \ - > images/postgres-backup.tag - cp images/postgres-backup.tag ../ferretdb/images/ - rm -f images/postgres-backup.json diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 764f1b94..67e9b27c 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -58,15 +58,22 @@ more details: ### Backup parameters -| Name | Description | Value | -| ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ | -| `backup.enabled` | Enable pereiodic backups | `false` | -| `backup.s3Region` | The AWS S3 region where backups are stored | `us-east-1` | -| `backup.s3Bucket` | The S3 bucket used for storing backups | `s3.example.org/postgres-backups` | -| `backup.schedule` | Cron schedule for automated backups | `0 2 * * *` | -| `backup.cleanupStrategy` | The strategy for cleaning up old backups | `--keep-last=3 --keep-daily=3 --keep-within-weekly=1m` | -| `backup.s3AccessKey` | The access key for S3, used for authentication | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu` | -| `backup.s3SecretKey` | The secret key for S3, used for authentication | `ju3eum4dekeich9ahM1te8waeGai0oog` | -| `backup.resticPassword` | The password for Restic backup encryption | `ChaXoveekoh6eigh4siesheeda2quai0` | -| `resources` | Resources | `{}` | -| `resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `nano` | +| Name | Description | Value | +| ------------------------ | -------------------------------------------------------------------- | ----------------------------------- | +| `backup.enabled` | Enable pereiodic backups | `false` | +| `backup.schedule` | Cron schedule for automated backups | `0 2 * * * *` | +| `backup.retentionPolicy` | The retention policy | `30d` | +| `backup.destinationPath` | The path where to store the backup (i.e. s3://bucket/path/to/folder) | `s3://BUCKET_NAME/` | +| `backup.endpointURL` | Endpoint to be used to upload data to the cloud | `http://minio-gateway-service:9000` | +| `backup.s3AccessKey` | The access key for S3, used for authentication | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu` | +| `backup.s3SecretKey` | The secret key for S3, used for authentication | `ju3eum4dekeich9ahM1te8waeGai0oog` | + +### Bootstrap parameters + +| Name | Description | Value | +| ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| `bootstrap.enabled` | Restore cluster from backup | `false` | +| `bootstrap.recoveryTime` | Time stamp up to which recovery will proceed, expressed in RFC 3339 format, if empty, will restore latest | `""` | +| `bootstrap.oldName` | Name of cluster before deleting | `""` | +| `resources` | Resources | `{}` | +| `resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `micro` | diff --git a/packages/apps/postgres/images/postgres-backup.tag b/packages/apps/postgres/images/postgres-backup.tag deleted file mode 100644 index c06767fd..00000000 --- a/packages/apps/postgres/images/postgres-backup.tag +++ /dev/null @@ -1 +0,0 @@ -ghcr.io/cozystack/cozystack/postgres-backup:0.14.0@sha256:10179ed56457460d95cd5708db2a00130901255fa30c4dd76c65d2ef5622b61f diff --git a/packages/apps/postgres/images/postgres-backup/Dockerfile b/packages/apps/postgres/images/postgres-backup/Dockerfile deleted file mode 100644 index aa38c323..00000000 --- a/packages/apps/postgres/images/postgres-backup/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM alpine:3.20 -RUN apk add --no-cache postgresql16-client uuidgen restic diff --git a/packages/apps/postgres/templates/backup-cronjob.yaml b/packages/apps/postgres/templates/backup-cronjob.yaml deleted file mode 100644 index 233f020a..00000000 --- a/packages/apps/postgres/templates/backup-cronjob.yaml +++ /dev/null @@ -1,99 +0,0 @@ -{{- if .Values.backup.enabled }} -{{ $image := .Files.Get "images/backup.json" | fromJson }} - -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ .Release.Name }}-backup -spec: - schedule: "{{ .Values.backup.schedule }}" - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - backoffLimit: 2 - template: - metadata: - annotations: - checksum/config: {{ include (print $.Template.BasePath "/backup-script.yaml") . | sha256sum }} - checksum/secret: {{ include (print $.Template.BasePath "/backup-secret.yaml") . | sha256sum }} - spec: - imagePullSecrets: - - name: {{ .Release.Name }}-regsecret - restartPolicy: OnFailure - containers: - - name: pgdump - image: "{{ $.Files.Get "images/postgres-backup.tag" | trim }}" - command: - - /bin/sh - - /scripts/backup.sh - env: - - name: REPO_PREFIX - value: {{ required "s3Bucket is not specified!" .Values.backup.s3Bucket | quote }} - - name: CLEANUP_STRATEGY - value: {{ required "cleanupStrategy is not specified!" .Values.backup.cleanupStrategy | quote }} - - name: PGUSER - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-superuser - key: username - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-superuser - key: password - - name: PGHOST - value: {{ .Release.Name }}-rw - - name: PGPORT - value: "5432" - - name: PGDATABASE - value: postgres - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-backup - key: s3AccessKey - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-backup - key: s3SecretKey - - name: AWS_DEFAULT_REGION - value: {{ .Values.backup.s3Region }} - - name: RESTIC_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-backup - key: resticPassword - volumeMounts: - - mountPath: /scripts - name: scripts - - mountPath: /tmp - name: tmp - - mountPath: /.cache - name: cache - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - privileged: false - readOnlyRootFilesystem: true - runAsNonRoot: true - {{- include "postgresjobs.resources" . | nindent 12 }} - volumes: - - name: scripts - secret: - secretName: {{ .Release.Name }}-backup-script - - name: tmp - emptyDir: {} - - name: cache - emptyDir: {} - securityContext: - runAsNonRoot: true - runAsUser: 9000 - runAsGroup: 9000 - seccompProfile: - type: RuntimeDefault -{{- end }} diff --git a/packages/apps/postgres/templates/backup-script.yaml b/packages/apps/postgres/templates/backup-script.yaml deleted file mode 100644 index 362bdc01..00000000 --- a/packages/apps/postgres/templates/backup-script.yaml +++ /dev/null @@ -1,50 +0,0 @@ -{{- if .Values.backup.enabled }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Release.Name }}-backup-script -stringData: - backup.sh: | - #!/bin/sh - set -e - set -o pipefail - - JOB_ID="job-$(uuidgen|cut -f1 -d-)" - DB_LIST=$(psql -Atq -c 'SELECT datname FROM pg_catalog.pg_database;' | grep -v '^\(postgres\|app\|template.*\)$') - echo DB_LIST=$(echo "$DB_LIST" | shuf) # shuffle list - echo "Job ID: $JOB_ID" - echo "Target repo: $REPO_PREFIX" - echo "Cleanup strategy: $CLEANUP_STRATEGY" - echo "Start backup for:" - echo "$DB_LIST" - echo - echo "Backup started at `date +%Y-%m-%d\ %H:%M:%S`" - for db in $DB_LIST; do - ( - set -x - restic -r "s3:${REPO_PREFIX}/$db" cat config >/dev/null 2>&1 || \ - restic -r "s3:${REPO_PREFIX}/$db" init --repository-version 2 - restic -r "s3:${REPO_PREFIX}/$db" unlock --remove-all >/dev/null 2>&1 || true # no locks, k8s takes care of it - pg_dump -Z0 -Ft -d "$db" | \ - restic -r "s3:${REPO_PREFIX}/$db" backup --tag "$JOB_ID" --stdin --stdin-filename dump.tar - restic -r "s3:${REPO_PREFIX}/$db" tag --tag "$JOB_ID" --set "completed" - ) - done - echo "Backup finished at `date +%Y-%m-%d\ %H:%M:%S`" - - echo - echo "Run cleanup:" - echo - - echo "Cleanup started at `date +%Y-%m-%d\ %H:%M:%S`" - for db in $DB_LIST; do - ( - set -x - restic forget -r "s3:${REPO_PREFIX}/$db" --group-by=tags --keep-tag "completed" # keep completed snapshots only - restic forget -r "s3:${REPO_PREFIX}/$db" --group-by=tags $CLEANUP_STRATEGY - restic prune -r "s3:${REPO_PREFIX}/$db" - ) - done - echo "Cleanup finished at `date +%Y-%m-%d\ %H:%M:%S`" -{{- end }} diff --git a/packages/apps/postgres/templates/backup-secret.yaml b/packages/apps/postgres/templates/backup-secret.yaml index be221e2f..3ae4879f 100644 --- a/packages/apps/postgres/templates/backup-secret.yaml +++ b/packages/apps/postgres/templates/backup-secret.yaml @@ -1,11 +1,10 @@ -{{- if .Values.backup.enabled }} +{{- if or .Values.backup.enabled .Values.bootstrap.enabled }} --- apiVersion: v1 kind: Secret metadata: - name: {{ .Release.Name }}-backup + name: {{ .Release.Name }}-s3-creds stringData: - s3AccessKey: {{ required "s3AccessKey is not specified!" .Values.backup.s3AccessKey }} - s3SecretKey: {{ required "s3SecretKey is not specified!" .Values.backup.s3SecretKey }} - resticPassword: {{ required "resticPassword is not specified!" .Values.backup.resticPassword }} + AWS_ACCESS_KEY_ID: {{ required "s3AccessKey is not specified!" .Values.backup.s3AccessKey | quote }} + AWS_SECRET_ACCESS_KEY: {{ required "s3SecretKey is not specified!" .Values.backup.s3SecretKey | quote }} {{- end }} diff --git a/packages/apps/postgres/templates/backup.yaml b/packages/apps/postgres/templates/backup.yaml new file mode 100644 index 00000000..7ed5321b --- /dev/null +++ b/packages/apps/postgres/templates/backup.yaml @@ -0,0 +1,12 @@ +{{- if .Values.backup.enabled }} +--- +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: {{ .Release.Name }} +spec: + schedule: {{ .Values.backup.schedule }} + backupOwnerReference: self + cluster: + name: {{ .Release.Name }} +{{- end }} diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 8218986c..3ce939e8 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -5,6 +5,43 @@ metadata: name: {{ .Release.Name }} spec: instances: {{ .Values.replicas }} + {{- if .Values.backup.enabled }} + backup: + barmanObjectStore: + destinationPath: {{ .Values.backup.destinationPath }} + endpointURL: {{ .Values.backup.endpointURL }} + s3Credentials: + accessKeyId: + name: {{ .Release.Name }}-s3-creds + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: {{ .Release.Name }}-s3-creds + key: AWS_SECRET_ACCESS_KEY + retentionPolicy: {{ .Values.backup.retentionPolicy }} + {{- end }} + + {{- if .Values.bootstrap.enabled }} + bootstrap: + recovery: + source: {{ .Values.bootstrap.oldName }} + {{- if .Values.bootstrap.recoveryTime }} + recoveryTarget: + targetTime: {{ .Values.bootstrap.recoveryTime }} + {{- end }} + externalClusters: + - name: {{ .Values.bootstrap.oldName }} + barmanObjectStore: + destinationPath: {{ .Values.backup.destinationPath }} + endpointURL: {{ .Values.backup.endpointURL }} + s3Credentials: + accessKeyId: + name: {{ .Release.Name }}-s3-creds + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: {{ .Release.Name }}-s3-creds + key: AWS_SECRET_ACCESS_KEY + {{- end }} + {{- if .Values.resources }} resources: {{- include "cozy-lib.resources.sanitize" (list .Values.resources $) | nindent 4 }} {{- else if ne .Values.resourcesPreset "none" }} diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 600e7675..4eaaccc6 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -65,25 +65,25 @@ "description": "Enable pereiodic backups", "default": false }, - "s3Region": { - "type": "string", - "description": "The AWS S3 region where backups are stored", - "default": "us-east-1" - }, - "s3Bucket": { - "type": "string", - "description": "The S3 bucket used for storing backups", - "default": "s3.example.org/postgres-backups" - }, "schedule": { "type": "string", "description": "Cron schedule for automated backups", - "default": "0 2 * * *" + "default": "0 2 * * * *" }, - "cleanupStrategy": { + "retentionPolicy": { "type": "string", - "description": "The strategy for cleaning up old backups", - "default": "--keep-last=3 --keep-daily=3 --keep-within-weekly=1m" + "description": "The retention policy", + "default": "30d" + }, + "destinationPath": { + "type": "string", + "description": "The path where to store the backup (i.e. s3://bucket/path/to/folder)", + "default": "s3://BUCKET_NAME/" + }, + "endpointURL": { + "type": "string", + "description": "Endpoint to be used to upload data to the cloud", + "default": "http://minio-gateway-service:9000" }, "s3AccessKey": { "type": "string", @@ -94,11 +94,26 @@ "type": "string", "description": "The secret key for S3, used for authentication", "default": "ju3eum4dekeich9ahM1te8waeGai0oog" + } + } + }, + "bootstrap": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Restore cluster from backup", + "default": false }, - "resticPassword": { + "recoveryTime": { "type": "string", - "description": "The password for Restic backup encryption", - "default": "ChaXoveekoh6eigh4siesheeda2quai0" + "description": "Time stamp up to which recovery will proceed, expressed in RFC 3339 format, if empty, will restore latest", + "default": "" + }, + "oldName": { + "type": "string", + "description": "Name of cluster before deleting", + "default": "" } } }, @@ -110,7 +125,7 @@ "resourcesPreset": { "type": "string", "description": "Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).", - "default": "nano" + "default": "micro" } } } \ No newline at end of file diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index 07618c56..d305c282 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -60,22 +60,32 @@ databases: {} ## @section Backup parameters ## @param backup.enabled Enable pereiodic backups -## @param backup.s3Region The AWS S3 region where backups are stored -## @param backup.s3Bucket The S3 bucket used for storing backups ## @param backup.schedule Cron schedule for automated backups -## @param backup.cleanupStrategy The strategy for cleaning up old backups +## @param backup.retentionPolicy The retention policy +## @param backup.destinationPath The path where to store the backup (i.e. s3://bucket/path/to/folder) +## @param backup.endpointURL Endpoint to be used to upload data to the cloud ## @param backup.s3AccessKey The access key for S3, used for authentication ## @param backup.s3SecretKey The secret key for S3, used for authentication -## @param backup.resticPassword The password for Restic backup encryption backup: enabled: false - s3Region: us-east-1 - s3Bucket: s3.example.org/postgres-backups - schedule: "0 2 * * *" - cleanupStrategy: "--keep-last=3 --keep-daily=3 --keep-within-weekly=1m" + retentionPolicy: 30d + destinationPath: s3://BUCKET_NAME/ + endpointURL: http://minio-gateway-service:9000 + schedule: "0 2 * * * *" s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog - resticPassword: ChaXoveekoh6eigh4siesheeda2quai0 + +## @section Bootstrap parameters + +## @param bootstrap.enabled Restore cluster from backup +## @param bootstrap.recoveryTime Time stamp up to which recovery will proceed, expressed in RFC 3339 format, if empty, will restore latest +## @param bootstrap.oldName Name of cluster before deleting +## +bootstrap: + enabled: false + # example: 2020-11-26 15:22:00.00000+00 + recoveryTime: "" + oldName: "" ## @param resources Resources resources: {} @@ -88,4 +98,4 @@ resources: {} # memory: 512Mi ## @param resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). -resourcesPreset: "nano" +resourcesPreset: "micro" diff --git a/packages/apps/versions_map b/packages/apps/versions_map index c7bb8276..6d9dc86c 100644 --- a/packages/apps/versions_map +++ b/packages/apps/versions_map @@ -87,7 +87,8 @@ postgres 0.10.1 93bdf411 postgres 0.11.0 f9f8bb2f postgres 0.12.0 6130f43d postgres 0.12.1 632224a3 -postgres 0.14.0 HEAD +postgres 0.14.0 62cb694d +postgres 0.15.0 HEAD rabbitmq 0.1.0 263e47be rabbitmq 0.2.0 53f2365e rabbitmq 0.3.0 6c5cf5bf