github-personal/cozystack
1
0
Fork 0
mirror of https://github.com/outbackdingo/cozystack.git synced 2026-01-27 10:18:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

[postgres] add backup and restore

Browse Source 
Signed-off-by: kklinch0 <kklinch0@gmail.com>
This commit is contained in:
kklinch0
2025-06-20 13:52:29 +03:00
committed by Andrei Kvapil
parent f0fc3238ca
commit 7c45335abb
14 changed files with 128 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@@ -9,7 +9,6 @@ build-deps:
build: build-deps
make -C packages/apps/http-cache image
make -C packages/apps/postgres image
make -C packages/apps/mysql image
make -C packages/apps/clickhouse image
make -C packages/apps/kubernetes image

2
packages/apps/postgres/Chart.yaml
View File

@@ -16,7 +16,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.14.0
version: 0.15.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

20
packages/apps/postgres/Makefile
View File

@@ -1,24 +1,4 @@
POSTGRES_BACKUP_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
include ../../../scripts/common-envs.mk
include ../../../scripts/package.mk
generate:
readme-generator -v values.yaml -s values.schema.json -r README.md
image:
docker buildx build images/postgres-backup \
--provenance false \
--builder=$(BUILDER) \
--platform=$(PLATFORM) \
--tag $(REGISTRY)/postgres-backup:$(call settag,$(POSTGRES_BACKUP_TAG)) \
--cache-from type=registry,ref=$(REGISTRY)/postgres-backup:latest \
--cache-to type=inline \
--metadata-file images/postgres-backup.json \
--push=$(PUSH) \
--label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \
--load=$(LOAD)
echo "$(REGISTRY)/postgres-backup:$(call settag,$(POSTGRES_BACKUP_TAG))@$$(yq e '."containerimage.digest"' images/postgres-backup.json -o json -r)" \
> images/postgres-backup.tag
cp images/postgres-backup.tag ../ferretdb/images/
rm -f images/postgres-backup.json

31
packages/apps/postgres/README.md
View File

@@ -58,15 +58,22 @@ more details:
### Backup parameters
| Name | Description | Value |
| ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ |
| `backup.enabled` | Enable pereiodic backups | `false` |
| `backup.s3Region` | The AWS S3 region where backups are stored | `us-east-1` |
| `backup.s3Bucket` | The S3 bucket used for storing backups | `s3.example.org/postgres-backups` |
| `backup.schedule` | Cron schedule for automated backups | `0 2 * * *` |
| `backup.cleanupStrategy` | The strategy for cleaning up old backups | `--keep-last=3 --keep-daily=3 --keep-within-weekly=1m` |
| `backup.s3AccessKey` | The access key for S3, used for authentication | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu` |
| `backup.s3SecretKey` | The secret key for S3, used for authentication | `ju3eum4dekeich9ahM1te8waeGai0oog` |
| `backup.resticPassword` | The password for Restic backup encryption | `ChaXoveekoh6eigh4siesheeda2quai0` |
| `resources` | Resources | `{}` |
| `resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `nano` |
| Name | Description | Value |
| ------------------------ | -------------------------------------------------------------------- | ----------------------------------- |
| `backup.enabled` | Enable pereiodic backups | `false` |
| `backup.schedule` | Cron schedule for automated backups | `0 2 * * * *` |
| `backup.retentionPolicy` | The retention policy | `30d` |
| `backup.destinationPath` | The path where to store the backup (i.e. s3://bucket/path/to/folder) | `s3://BUCKET_NAME/` |
| `backup.endpointURL` | Endpoint to be used to upload data to the cloud | `http://minio-gateway-service:9000` |
| `backup.s3AccessKey` | The access key for S3, used for authentication | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu` |
| `backup.s3SecretKey` | The secret key for S3, used for authentication | `ju3eum4dekeich9ahM1te8waeGai0oog` |
### Bootstrap parameters
| Name | Description | Value |
| ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
| `bootstrap.enabled` | Restore cluster from backup | `false` |
| `bootstrap.recoveryTime` | Time stamp up to which recovery will proceed, expressed in RFC 3339 format, if empty, will restore latest | `""` |
| `bootstrap.oldName` | Name of cluster before deleting | `""` |
| `resources` | Resources | `{}` |
| `resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `micro` |

1
packages/apps/postgres/images/postgres-backup.tag
View File

@@ -1 +0,0 @@
ghcr.io/cozystack/cozystack/postgres-backup:0.14.0@sha256:10179ed56457460d95cd5708db2a00130901255fa30c4dd76c65d2ef5622b61f

2
packages/apps/postgres/images/postgres-backup/Dockerfile
View File

@@ -1,2 +0,0 @@
FROM alpine:3.20
RUN apk add --no-cache postgresql16-client uuidgen restic

99
packages/apps/postgres/templates/backup-cronjob.yaml
View File

@@ -1,99 +0,0 @@
{{- if .Values.backup.enabled }}
{{ $image := .Files.Get "images/backup.json" | fromJson }}
apiVersion: batch/v1
kind: CronJob
metadata:
name: {{ .Release.Name }}-backup
spec:
schedule: "{{ .Values.backup.schedule }}"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 2
template:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/backup-script.yaml") . | sha256sum }}
checksum/secret: {{ include (print $.Template.BasePath "/backup-secret.yaml") . | sha256sum }}
spec:
imagePullSecrets:
- name: {{ .Release.Name }}-regsecret
restartPolicy: OnFailure
containers:
- name: pgdump
image: "{{ $.Files.Get "images/postgres-backup.tag" | trim }}"
command:
- /bin/sh
- /scripts/backup.sh
env:
- name: REPO_PREFIX
value: {{ required "s3Bucket is not specified!" .Values.backup.s3Bucket | quote }}
- name: CLEANUP_STRATEGY
value: {{ required "cleanupStrategy is not specified!" .Values.backup.cleanupStrategy | quote }}
- name: PGUSER
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-superuser
key: username
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-superuser
key: password
- name: PGHOST
value: {{ .Release.Name }}-rw
- name: PGPORT
value: "5432"
- name: PGDATABASE
value: postgres
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-backup
key: s3AccessKey
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-backup
key: s3SecretKey
- name: AWS_DEFAULT_REGION
value: {{ .Values.backup.s3Region }}
- name: RESTIC_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-backup
key: resticPassword
volumeMounts:
- mountPath: /scripts
name: scripts
- mountPath: /tmp
name: tmp
- mountPath: /.cache
name: cache
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
{{- include "postgresjobs.resources" . | nindent 12 }}
volumes:
- name: scripts
secret:
secretName: {{ .Release.Name }}-backup-script
- name: tmp
emptyDir: {}
- name: cache
emptyDir: {}
securityContext:
runAsNonRoot: true
runAsUser: 9000
runAsGroup: 9000
seccompProfile:
type: RuntimeDefault
{{- end }}

50
packages/apps/postgres/templates/backup-script.yaml
View File

@@ -1,50 +0,0 @@
{{- if .Values.backup.enabled }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-backup-script
stringData:
backup.sh: |
#!/bin/sh
set -e
set -o pipefail
JOB_ID="job-$(uuidgen|cut -f1 -d-)"
DB_LIST=$(psql -Atq -c 'SELECT datname FROM pg_catalog.pg_database;' | grep -v '^\(postgres\|app\|template.*\)$')
echo DB_LIST=$(echo "$DB_LIST" | shuf) # shuffle list
echo "Job ID: $JOB_ID"
echo "Target repo: $REPO_PREFIX"
echo "Cleanup strategy: $CLEANUP_STRATEGY"
echo "Start backup for:"
echo "$DB_LIST"
echo
echo "Backup started at `date +%Y-%m-%d\ %H:%M:%S`"
for db in $DB_LIST; do
(
set -x
restic -r "s3:${REPO_PREFIX}/$db" cat config >/dev/null 2>&1 || \
restic -r "s3:${REPO_PREFIX}/$db" init --repository-version 2
restic -r "s3:${REPO_PREFIX}/$db" unlock --remove-all >/dev/null 2>&1 || true # no locks, k8s takes care of it
pg_dump -Z0 -Ft -d "$db" | \
restic -r "s3:${REPO_PREFIX}/$db" backup --tag "$JOB_ID" --stdin --stdin-filename dump.tar
restic -r "s3:${REPO_PREFIX}/$db" tag --tag "$JOB_ID" --set "completed"
)
done
echo "Backup finished at `date +%Y-%m-%d\ %H:%M:%S`"
echo
echo "Run cleanup:"
echo
echo "Cleanup started at `date +%Y-%m-%d\ %H:%M:%S`"
for db in $DB_LIST; do
(
set -x
restic forget -r "s3:${REPO_PREFIX}/$db" --group-by=tags --keep-tag "completed" # keep completed snapshots only
restic forget -r "s3:${REPO_PREFIX}/$db" --group-by=tags $CLEANUP_STRATEGY
restic prune -r "s3:${REPO_PREFIX}/$db"
)
done
echo "Cleanup finished at `date +%Y-%m-%d\ %H:%M:%S`"
{{- end }}

9
packages/apps/postgres/templates/backup-secret.yaml
View File

@@ -1,11 +1,10 @@
{{- if .Values.backup.enabled }}
{{- if or .Values.backup.enabled .Values.bootstrap.enabled }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-backup
name: {{ .Release.Name }}-s3-creds
stringData:
s3AccessKey: {{ required "s3AccessKey is not specified!" .Values.backup.s3AccessKey }}
s3SecretKey: {{ required "s3SecretKey is not specified!" .Values.backup.s3SecretKey }}
resticPassword: {{ required "resticPassword is not specified!" .Values.backup.resticPassword }}
AWS_ACCESS_KEY_ID: {{ required "s3AccessKey is not specified!" .Values.backup.s3AccessKey | quote }}
AWS_SECRET_ACCESS_KEY: {{ required "s3SecretKey is not specified!" .Values.backup.s3SecretKey | quote }}
{{- end }}

12
packages/apps/postgres/templates/backup.yaml Normal file
View File

@@ -0,0 +1,12 @@
{{- if .Values.backup.enabled }}
---
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: {{ .Release.Name }}
spec:
schedule: {{ .Values.backup.schedule }}
backupOwnerReference: self
cluster:
name: {{ .Release.Name }}
{{- end }}

37
packages/apps/postgres/templates/db.yaml
View File

@@ -5,6 +5,43 @@ metadata:
name: {{ .Release.Name }}
spec:
instances: {{ .Values.replicas }}
{{- if .Values.backup.enabled }}
backup:
barmanObjectStore:
destinationPath: {{ .Values.backup.destinationPath }}
endpointURL: {{ .Values.backup.endpointURL }}
s3Credentials:
accessKeyId:
name: {{ .Release.Name }}-s3-creds
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: {{ .Release.Name }}-s3-creds
key: AWS_SECRET_ACCESS_KEY
retentionPolicy: {{ .Values.backup.retentionPolicy }}
{{- end }}
{{- if .Values.bootstrap.enabled }}
bootstrap:
recovery:
source: {{ .Values.bootstrap.oldName }}
{{- if .Values.bootstrap.recoveryTime }}
recoveryTarget:
targetTime: {{ .Values.bootstrap.recoveryTime }}
{{- end }}
externalClusters:
- name: {{ .Values.bootstrap.oldName }}
barmanObjectStore:
destinationPath: {{ .Values.backup.destinationPath }}
endpointURL: {{ .Values.backup.endpointURL }}
s3Credentials:
accessKeyId:
name: {{ .Release.Name }}-s3-creds
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: {{ .Release.Name }}-s3-creds
key: AWS_SECRET_ACCESS_KEY
{{- end }}
{{- if .Values.resources }}
resources: {{- include "cozy-lib.resources.sanitize" (list .Values.resources $) | nindent 4 }}
{{- else if ne .Values.resourcesPreset "none" }}

51
packages/apps/postgres/values.schema.json
View File

@@ -65,25 +65,25 @@
"description": "Enable pereiodic backups",
"default": false
},
"s3Region": {
"type": "string",
"description": "The AWS S3 region where backups are stored",
"default": "us-east-1"
},
"s3Bucket": {
"type": "string",
"description": "The S3 bucket used for storing backups",
"default": "s3.example.org/postgres-backups"
},
"schedule": {
"type": "string",
"description": "Cron schedule for automated backups",
"default": "0 2 * * *"
"default": "0 2 * * * *"
},
"cleanupStrategy": {
"retentionPolicy": {
"type": "string",
"description": "The strategy for cleaning up old backups",
"default": "--keep-last=3 --keep-daily=3 --keep-within-weekly=1m"
"description": "The retention policy",
"default": "30d"
},
"destinationPath": {
"type": "string",
"description": "The path where to store the backup (i.e. s3://bucket/path/to/folder)",
"default": "s3://BUCKET_NAME/"
},
"endpointURL": {
"type": "string",
"description": "Endpoint to be used to upload data to the cloud",
"default": "http://minio-gateway-service:9000"
},
"s3AccessKey": {
"type": "string",
@@ -94,11 +94,26 @@
"type": "string",
"description": "The secret key for S3, used for authentication",
"default": "ju3eum4dekeich9ahM1te8waeGai0oog"
}
}
},
"bootstrap": {
"type": "object",
"properties": {
"enabled": {
"type": "boolean",
"description": "Restore cluster from backup",
"default": false
},
"resticPassword": {
"recoveryTime": {
"type": "string",
"description": "The password for Restic backup encryption",
"default": "ChaXoveekoh6eigh4siesheeda2quai0"
"description": "Time stamp up to which recovery will proceed, expressed in RFC 3339 format, if empty, will restore latest",
"default": ""
},
"oldName": {
"type": "string",
"description": "Name of cluster before deleting",
"default": ""
}
}
},
@@ -110,7 +125,7 @@
"resourcesPreset": {
"type": "string",
"description": "Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).",
"default": "nano"
"default": "micro"
}
}
}

30
packages/apps/postgres/values.yaml
View File

@@ -60,22 +60,32 @@ databases: {}
## @section Backup parameters
## @param backup.enabled Enable pereiodic backups
## @param backup.s3Region The AWS S3 region where backups are stored
## @param backup.s3Bucket The S3 bucket used for storing backups
## @param backup.schedule Cron schedule for automated backups
## @param backup.cleanupStrategy The strategy for cleaning up old backups
## @param backup.retentionPolicy The retention policy
## @param backup.destinationPath The path where to store the backup (i.e. s3://bucket/path/to/folder)
## @param backup.endpointURL Endpoint to be used to upload data to the cloud
## @param backup.s3AccessKey The access key for S3, used for authentication
## @param backup.s3SecretKey The secret key for S3, used for authentication
## @param backup.resticPassword The password for Restic backup encryption
backup:
enabled: false
s3Region: us-east-1
s3Bucket: s3.example.org/postgres-backups
schedule: "0 2 * * *"
cleanupStrategy: "--keep-last=3 --keep-daily=3 --keep-within-weekly=1m"
retentionPolicy: 30d
destinationPath: s3://BUCKET_NAME/
endpointURL: http://minio-gateway-service:9000
schedule: "0 2 * * * *"
s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
resticPassword: ChaXoveekoh6eigh4siesheeda2quai0
## @section Bootstrap parameters
## @param bootstrap.enabled Restore cluster from backup
## @param bootstrap.recoveryTime Time stamp up to which recovery will proceed, expressed in RFC 3339 format, if empty, will restore latest
## @param bootstrap.oldName Name of cluster before deleting
##
bootstrap:
enabled: false
# example: 2020-11-26 15:22:00.00000+00
recoveryTime: ""
oldName: ""
## @param resources Resources
resources: {}
@@ -88,4 +98,4 @@ resources: {}
# memory: 512Mi
## @param resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).
resourcesPreset: "nano"
resourcesPreset: "micro"

3
packages/apps/versions_map
View File

@@ -87,7 +87,8 @@ postgres 0.10.1 93bdf411
postgres 0.11.0 f9f8bb2f
postgres 0.12.0 6130f43d
postgres 0.12.1 632224a3
postgres 0.14.0 HEAD
postgres 0.14.0 62cb694d
postgres 0.15.0 HEAD
rabbitmq 0.1.0 263e47be
rabbitmq 0.2.0 53f2365e
rabbitmq 0.3.0 6c5cf5bf