diff --git a/kube/deploy/apps/atuin/ks.yaml b/kube/deploy/apps/atuin/ks.yaml index e67a6181..c622c582 100644 --- a/kube/deploy/apps/atuin/ks.yaml +++ b/kube/deploy/apps/atuin/ks.yaml @@ -20,6 +20,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "atuin" diff --git a/kube/deploy/apps/authentik/ks.yaml b/kube/deploy/apps/authentik/ks.yaml index 1cf366a8..b010d6f4 100644 --- a/kube/deploy/apps/authentik/ks.yaml +++ b/kube/deploy/apps/authentik/ks.yaml @@ -36,6 +36,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "authentik" diff --git a/kube/deploy/apps/firefly/ks.yaml b/kube/deploy/apps/firefly/ks.yaml index 66036332..917ddeb6 100644 --- a/kube/deploy/apps/firefly/ks.yaml +++ b/kube/deploy/apps/firefly/ks.yaml @@ -22,6 +22,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "firefly" diff --git a/kube/deploy/apps/gotosocial/ks.yaml b/kube/deploy/apps/gotosocial/ks.yaml index 8e495336..e323cf51 100644 --- a/kube/deploy/apps/gotosocial/ks.yaml +++ b/kube/deploy/apps/gotosocial/ks.yaml @@ -20,6 +20,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "gotosocial" @@ -39,4 +40,4 @@ spec: name: not-used target: group: postgresql.cnpg.io/v1 - kind: Cluster \ No newline at end of file + kind: Cluster diff --git a/kube/deploy/apps/joplin/ks.yaml b/kube/deploy/apps/joplin/ks.yaml index 2d5221a0..715ec1a1 100644 --- a/kube/deploy/apps/joplin/ks.yaml +++ b/kube/deploy/apps/joplin/ks.yaml @@ -20,6 +20,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "joplin" diff --git a/kube/deploy/apps/miniflux/ks.yaml b/kube/deploy/apps/miniflux/ks.yaml index 2a05b418..7959fd06 100644 --- a/kube/deploy/apps/miniflux/ks.yaml +++ b/kube/deploy/apps/miniflux/ks.yaml @@ -22,6 +22,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "miniflux" diff --git a/kube/deploy/apps/paperless-ngx/ks.yaml b/kube/deploy/apps/paperless-ngx/ks.yaml index 88eebab1..50a3290c 100644 --- a/kube/deploy/apps/paperless-ngx/ks.yaml +++ b/kube/deploy/apps/paperless-ngx/ks.yaml @@ -18,6 +18,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "paperless-ngx" @@ -58,4 +59,4 @@ spec: group: "" version: "v1" kind: "PersistentVolume" - name: "pg-paperless-ngx-wal-nfs" \ No newline at end of file + name: "pg-paperless-ngx-wal-nfs" diff --git a/kube/deploy/apps/piped/ks.yaml b/kube/deploy/apps/piped/ks.yaml index f67173af..4de9713c 100644 --- a/kube/deploy/apps/piped/ks.yaml +++ b/kube/deploy/apps/piped/ks.yaml @@ -20,6 +20,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "piped" diff --git a/kube/deploy/apps/soft-serve/ks.yaml b/kube/deploy/apps/soft-serve/ks.yaml index 3a9aa735..e00244fd 100644 --- a/kube/deploy/apps/soft-serve/ks.yaml +++ b/kube/deploy/apps/soft-serve/ks.yaml @@ -22,6 +22,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "soft-serve" diff --git a/kube/deploy/apps/zipline/ks.yaml b/kube/deploy/apps/zipline/ks.yaml index f5cb3acc..11e402a5 100644 --- a/kube/deploy/apps/zipline/ks.yaml +++ b/kube/deploy/apps/zipline/ks.yaml @@ -19,6 +19,7 @@ spec: dependsOn: - name: 1-core-db-pg-app - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s postBuild: substitute: PG_APP_NAME: &app "zipline" diff --git a/kube/deploy/core/db/pg/clusters/template/crunchy.yaml b/kube/deploy/core/db/pg/clusters/template/crunchy.yaml index 944dcc32..850e89ec 100644 --- a/kube/deploy/core/db/pg/clusters/template/crunchy.yaml +++ b/kube/deploy/core/db/pg/clusters/template/crunchy.yaml @@ -44,6 +44,8 @@ spec: configuration: &brcfg [secret: {name: "pg-${PG_APP_NAME}-secrets"}] global: &brflag archive-timeout: "60" # sends WAL archive every X seconds + compress-type: "bz2" + compress-level: "9" repo1-retention-full-type: "time" repo1-retention-full: "5" repo1-retention-diff: "30" @@ -53,9 +55,15 @@ spec: repo2-path: "/${PG_APP_NAME}" repo2-s3-uri-style: "path" repo2-retention-full-type: "time" - repo2-retention-full: "5" - repo2-retention-diff: "30" + repo2-retention-full: "2" + repo2-retention-diff: "7" repo2-cipher-type: "aes-256-cbc" + repo3-bundle: "y" + repo3-block: "y" + repo3-s3-uri-style: "path" + repo3-retention-full-type: "time" + repo3-retention-full: "5" + repo3-retention-diff: "30" repos: - name: "repo1" # NFS volume: &nfs @@ -79,14 +87,23 @@ spec: full: "30 6 * * 1" # every Monday at 06:30 differential: "30 6 * * 0,2-6" # every day at 06:30 except Monday incremental: "30 1-5,7-23 * * *" # every hour except 06:30 - dataSource: - pgbackrest: - stanza: "db" - configuration: *brcfg - global: *brflag - repo: - name: "repo2" - s3: *r2 + - name: "repo3" # Ceph RGW in-cluster + s3: &rgw + endpoint: "rook-ceph-rgw-${CLUSTER_NAME}.rook-ceph.svc.cluster.local:6953" + bucket: "pg-${PG_APP_NAME}" + region: "us-east-1" + schedules: # times staggered to avoid NFS schedule causing failed jobs due to locks + full: "15 6 * * 1" # every Monday at 06:15 + differential: "15 6 * * 0,2-6" # every day at 06:15 except Monday + incremental: "15 1-5,7-23 * * *" # every hour except 06:15 + # dataSource: + # pgbackrest: + # stanza: "db" + # configuration: *brcfg + # global: *brflag + # repo: + # name: "repo3" + # s3: *rgw proxy: pgBouncer: port: 5432 diff --git a/kube/deploy/core/db/pg/clusters/template/s3.yaml b/kube/deploy/core/db/pg/clusters/template/s3.yaml index 006a925d..ef2b9657 100644 --- a/kube/deploy/core/db/pg/clusters/template/s3.yaml +++ b/kube/deploy/core/db/pg/clusters/template/s3.yaml @@ -7,3 +7,37 @@ metadata: spec: bucketName: "pg-${PG_APP_NAME}" storageClassName: "rgw-${CLUSTER_NAME}" +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: &name "pg-${PG_APP_NAME}-s3" + namespace: "${PG_APP_NS}" + # name: "test" +spec: + refreshInterval: "1m" + secretStoreRef: + kind: "ClusterSecretStore" + name: "kubernetes" + target: + name: "pg-${PG_APP_NAME}-s3-crunchy" + creationPolicy: "Owner" + deletionPolicy: "Retain" + template: + type: "Opaque" + data: + s3.conf: | + [global] + repo3-s3-key={{ .AWS_ACCESS_KEY_ID }} + repo3-s3-key-secret={{ .AWS_SECRET_ACCESS_KEY }} + data: + - secretKey: &key "AWS_ACCESS_KEY_ID" + remoteRef: &src + key: *name + property: *key + decodingStrategy: "Auto" + - secretKey: &key "AWS_SECRET_ACCESS_KEY" + remoteRef: + <<: *src + property: *key