From 6b2c17b2c8275f20eeef879e47c5a581ea7184da Mon Sep 17 00:00:00 2001 From: JJGadgets Date: Wed, 1 May 2024 04:38:05 +0800 Subject: [PATCH] fix(pg/clusters): R2 bucket per cluster, use MinIO over NFS for NAS WALs --- .../core/db/pg/clusters/template/crunchy.yaml | 44 +++++++++++++------ .../db/pg/clusters/template/dump-local.yaml | 5 ++- kube/deploy/core/db/pg/ks.yaml | 16 +++++-- .../core/storage/minio-nas/app/netpol.yaml | 42 ++++++++++++++++++ .../core/storage/volsync/app/netpol.yaml | 7 +++ 5 files changed, 95 insertions(+), 19 deletions(-) create mode 100644 kube/deploy/core/storage/minio-nas/app/netpol.yaml diff --git a/kube/deploy/core/db/pg/clusters/template/crunchy.yaml b/kube/deploy/core/db/pg/clusters/template/crunchy.yaml index 194f51ea..a76dd7cd 100644 --- a/kube/deploy/core/db/pg/clusters/template/crunchy.yaml +++ b/kube/deploy/core/db/pg/clusters/template/crunchy.yaml @@ -43,20 +43,28 @@ spec: egress.home.arpa/apiserver: "allow" egress.home.arpa/r2: "allow" s3.home.arpa/store: "rgw-${CLUSTER_NAME}" + s3.home.arpa/minio-nas: "allow" configuration: &brcfg - secret: name: "pg-${PG_APP_NAME}-secrets" - secret: name: "pg-${PG_APP_NAME}-s3-crunchy" + #manual: + # repoName: "repo2" + # options: ["--type=full", "--annotation=reason=change-R2-buckets", "--archive-copy", "--checksum-page"] global: &brflag archive-timeout: "60" compress-type: "bz2" compress-level: "9" delta: "y" + repo1-bundle: "y" + repo1-block: "y" repo1-retention-full-type: "time" - repo1-retention-full: "5" - repo1-retention-diff: "30" - repo1-path: "/pgbackrest/repo1/${PG_APP_NAME}" + repo1-retention-full: "2" + repo1-retention-diff: "7" + #repo1-path: "/pgbackrest/repo1/${PG_APP_NAME}" # NFS + repo1-path: "/${PG_APP_NAME}" + repo1-s3-uri-style: "path" repo2-bundle: "y" repo2-block: "y" repo2-path: "/${PG_APP_NAME}" @@ -84,22 +92,25 @@ spec: differential: "15 6 * * 0,2-6" # every day at 06:15 except Monday incremental: "15 1-5,7-23 * * *" # every hour except 06:15 - name: "repo1" # NFS - volume: &nfs - volumeClaimSpec: - storageClassName: "pg-${PG_APP_NAME}-wal-nfs" - volumeName: "pg-${PG_APP_NAME}-wal-nfs" - accessModes: ["ReadWriteMany"] - resources: - requests: - storage: "1Mi" # doesn't matter with NFS + s3: &minio + endpoint: "${APP_DNS_MINIO_NAS_S3}" + bucket: "${SECRET_PGBACKREST_WAL_MINIO_BUCKET}" + region: "us-east-1" + #volume: &nfs + # volumeClaimSpec: + # storageClassName: "pg-${PG_APP_NAME}-wal-nfs" + # volumeName: "pg-${PG_APP_NAME}-wal-nfs" + # accessModes: ["ReadWriteMany"] + # resources: + # requests: + # storage: "1Mi" # doesn't matter with NFS schedules: # more aggressive for NAS (over NFS) full: "0 6 * * 1" # every Monday at 06:00 differential: "0 6 * * 2-6" # every day at 06:00 except Monday - incremental: "0 1-5,7-23 * * *" # every hour except 06:00 - name: "repo2" # Cloudflare R2 s3: &r2 endpoint: "${SECRET_PGBACKREST_WAL_R2_ENDPOINT}" - bucket: "${SECRET_PGBACKREST_WAL_R2_BUCKET}" + bucket: "${SECRET_PGBACKREST_WAL_R2_BUCKET}-${PG_APP_NAME}" region: "us-east-1" schedules: # times staggered to avoid NFS schedule causing failed jobs due to locks full: "30 6 * * 1" # every Monday at 06:30 @@ -111,7 +122,10 @@ spec: global: *brflag repo: name: "repo2" - s3: *r2 + s3: + endpoint: "${SECRET_PGBACKREST_WAL_R2_ENDPOINT}" + bucket: "${SECRET_PGBACKREST_WAL_R2_BUCKET}" + region: "us-east-1" proxy: pgBouncer: port: 5432 @@ -140,6 +154,8 @@ type: Opaque stringData: s3.conf: | [global] + repo1-s3-key=${SECRET_PGBACKREST_WAL_MINIO_ID} + repo1-s3-key-secret=${SECRET_PGBACKREST_WAL_MINIO_KEY} repo2-s3-key=${SECRET_PGBACKREST_WAL_R2_ID} repo2-s3-key-secret=${SECRET_PGBACKREST_WAL_R2_KEY} encryption.conf: | diff --git a/kube/deploy/core/db/pg/clusters/template/dump-local.yaml b/kube/deploy/core/db/pg/clusters/template/dump-local.yaml index 5a2d6379..ce2e75a2 100644 --- a/kube/deploy/core/db/pg/clusters/template/dump-local.yaml +++ b/kube/deploy/core/db/pg/clusters/template/dump-local.yaml @@ -5,9 +5,10 @@ metadata: name: "pg-${PG_APP_NAME}-${PG_DB_NAME}-dump-local" namespace: "${PG_APP_NS}" labels: &labels - postgresql: &pg "pg-${PG_APP_NAME}" - app.kubernetes.io/instance: *pg app.kubernetes.io/name: "pg-dump-local" + app.kubernetes.io/instance: &pg "pg-${PG_APP_NAME}" + postgresql: *pg # CNPG + postgres-operator.crunchydata.com/cluster: *pg kustomize.toolkit.fluxcd.io/force: "Enabled" spec: schedule: "0 6 * * *" diff --git a/kube/deploy/core/db/pg/ks.yaml b/kube/deploy/core/db/pg/ks.yaml index a77a8628..97c1a945 100644 --- a/kube/deploy/core/db/pg/ks.yaml +++ b/kube/deploy/core/db/pg/ks.yaml @@ -12,11 +12,21 @@ spec: wait: false dependsOn: - name: 1-core-storage-csi-driver-nfs-app + - name: 1-core-storage-democratic-csi-local-hostpath + - name: 1-core-secrets-es-k8s healthChecks: - - name: crunchy-pgo - namespace: crunchy-pgo + - apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease - apiVersion: helm.toolkit.fluxcd.io/v2beta1 + name: crunchy-pgo + namespace: crunchy-pgo + - apiVersion: apps/v1 + kind: Deployment + name: minio-nas + namespace: minio-nas + - apiVersion: apps/v1 + kind: Deployment + name: rook-ceph-rgw-biohazard-a + namespace: rook-ceph --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/kube/deploy/core/storage/minio-nas/app/netpol.yaml b/kube/deploy/core/storage/minio-nas/app/netpol.yaml new file mode 100644 index 00000000..c827b9b2 --- /dev/null +++ b/kube/deploy/core/storage/minio-nas/app/netpol.yaml @@ -0,0 +1,42 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: &app minio-nas + namespace: *app +spec: + endpointSelector: {} + ingress: + # same namespace + - fromEndpoints: + - matchLabels: + io.kubernetes.pod.namespace: *app + # allow HTTPS traffic in-cluster + - fromEndpoints: + - matchLabels: + s3.home.arpa/minio-nas: allow + matchExpressions: + - key: io.kubernetes.pod.namespace + operator: Exists + toPorts: + - ports: + - port: "9000" +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumclusterwidenetworkpolicy_v2.json +apiVersion: cilium.io/v2 +kind: CiliumClusterwideNetworkPolicy +metadata: + name: &app minio-nas-in-cluster +spec: + endpointSelector: + matchLabels: + s3.home.arpa/minio-nas: allow + egress: + - toEndpoints: + - matchLabels: + io.kubernetes.pod.namespace: minio-nas + app.kubernetes.io/name: minio-nas + toPorts: + - ports: + - port: "9000" diff --git a/kube/deploy/core/storage/volsync/app/netpol.yaml b/kube/deploy/core/storage/volsync/app/netpol.yaml index 65d463e0..92f84943 100644 --- a/kube/deploy/core/storage/volsync/app/netpol.yaml +++ b/kube/deploy/core/storage/volsync/app/netpol.yaml @@ -35,6 +35,10 @@ spec: endpointSelector: matchLabels: app.kubernetes.io/created-by: volsync + ingress: + - fromEndpoints: + - matchLabels: + prometheus: "kps" egress: # allow all to public Internet - toEntities: @@ -51,3 +55,6 @@ spec: toPorts: - ports: - port: "6953" + - toEndpoints: + - matchLabels: + rgw: "${CLUSTER_NAME}"