scottk/Biohazard
1
0
Fork 0
mirror of https://github.com/outbackdingo/Biohazard.git synced 2026-03-22 03:39:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(pg/clusters): R2 bucket per cluster, use MinIO over NFS for NAS WALs

Browse Source
This commit is contained in:
JJGadgets
2024-05-01 04:38:05 +08:00
parent 239d84f345
commit 6b2c17b2c8
5 changed files with 95 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
kube/deploy/core/db/pg/clusters/template/crunchy.yaml
View File

@@ -43,20 +43,28 @@ spec:
egress.home.arpa/apiserver: "allow"
egress.home.arpa/r2: "allow"
s3.home.arpa/store: "rgw-${CLUSTER_NAME}"
s3.home.arpa/minio-nas: "allow"
configuration: &brcfg
- secret:
name: "pg-${PG_APP_NAME}-secrets"
- secret:
name: "pg-${PG_APP_NAME}-s3-crunchy"
#manual:
# repoName: "repo2"
# options: ["--type=full", "--annotation=reason=change-R2-buckets", "--archive-copy", "--checksum-page"]
global: &brflag
archive-timeout: "60"
compress-type: "bz2"
compress-level: "9"
delta: "y"
repo1-bundle: "y"
repo1-block: "y"
repo1-retention-full-type: "time"
repo1-retention-full: "5"
repo1-retention-diff: "30"
repo1-path: "/pgbackrest/repo1/${PG_APP_NAME}"
repo1-retention-full: "2"
repo1-retention-diff: "7"
#repo1-path: "/pgbackrest/repo1/${PG_APP_NAME}" # NFS
repo1-path: "/${PG_APP_NAME}"
repo1-s3-uri-style: "path"
repo2-bundle: "y"
repo2-block: "y"
repo2-path: "/${PG_APP_NAME}"
@@ -84,22 +92,25 @@ spec:
differential: "15 6 * * 0,2-6" # every day at 06:15 except Monday
incremental: "15 1-5,7-23 * * *" # every hour except 06:15
- name: "repo1" # NFS
volume: &nfs
volumeClaimSpec:
storageClassName: "pg-${PG_APP_NAME}-wal-nfs"
volumeName: "pg-${PG_APP_NAME}-wal-nfs"
accessModes: ["ReadWriteMany"]
resources:
requests:
storage: "1Mi" # doesn't matter with NFS
s3: &minio
endpoint: "${APP_DNS_MINIO_NAS_S3}"
bucket: "${SECRET_PGBACKREST_WAL_MINIO_BUCKET}"
region: "us-east-1"
#volume: &nfs
# volumeClaimSpec:
# storageClassName: "pg-${PG_APP_NAME}-wal-nfs"
# volumeName: "pg-${PG_APP_NAME}-wal-nfs"
# accessModes: ["ReadWriteMany"]
# resources:
# requests:
# storage: "1Mi" # doesn't matter with NFS
schedules: # more aggressive for NAS (over NFS)
full: "0 6 * * 1" # every Monday at 06:00
differential: "0 6 * * 2-6" # every day at 06:00 except Monday
incremental: "0 1-5,7-23 * * *" # every hour except 06:00
- name: "repo2" # Cloudflare R2
s3: &r2
endpoint: "${SECRET_PGBACKREST_WAL_R2_ENDPOINT}"
bucket: "${SECRET_PGBACKREST_WAL_R2_BUCKET}"
bucket: "${SECRET_PGBACKREST_WAL_R2_BUCKET}-${PG_APP_NAME}"
region: "us-east-1"
schedules: # times staggered to avoid NFS schedule causing failed jobs due to locks
full: "30 6 * * 1" # every Monday at 06:30
@@ -111,7 +122,10 @@ spec:
global: *brflag
repo:
name: "repo2"
s3: *r2
s3:
endpoint: "${SECRET_PGBACKREST_WAL_R2_ENDPOINT}"
bucket: "${SECRET_PGBACKREST_WAL_R2_BUCKET}"
region: "us-east-1"
proxy:
pgBouncer:
port: 5432
@@ -140,6 +154,8 @@ type: Opaque
stringData:
s3.conf: |
[global]
repo1-s3-key=${SECRET_PGBACKREST_WAL_MINIO_ID}
repo1-s3-key-secret=${SECRET_PGBACKREST_WAL_MINIO_KEY}
repo2-s3-key=${SECRET_PGBACKREST_WAL_R2_ID}
repo2-s3-key-secret=${SECRET_PGBACKREST_WAL_R2_KEY}
encryption.conf: |

5
kube/deploy/core/db/pg/clusters/template/dump-local.yaml
View File

@@ -5,9 +5,10 @@ metadata:
name: "pg-${PG_APP_NAME}-${PG_DB_NAME}-dump-local"
namespace: "${PG_APP_NS}"
labels: &labels
postgresql: &pg "pg-${PG_APP_NAME}"
app.kubernetes.io/instance: *pg
app.kubernetes.io/name: "pg-dump-local"
app.kubernetes.io/instance: &pg "pg-${PG_APP_NAME}"
postgresql: *pg # CNPG
postgres-operator.crunchydata.com/cluster: *pg
kustomize.toolkit.fluxcd.io/force: "Enabled"
spec:
schedule: "0 6 * * *"

16
kube/deploy/core/db/pg/ks.yaml
View File

@@ -12,11 +12,21 @@ spec:
wait: false
dependsOn:
- name: 1-core-storage-csi-driver-nfs-app
- name: 1-core-storage-democratic-csi-local-hostpath
- name: 1-core-secrets-es-k8s
healthChecks:
- name: crunchy-pgo
namespace: crunchy-pgo
- apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
apiVersion: helm.toolkit.fluxcd.io/v2beta1
name: crunchy-pgo
namespace: crunchy-pgo
- apiVersion: apps/v1
kind: Deployment
name: minio-nas
namespace: minio-nas
- apiVersion: apps/v1
kind: Deployment
name: rook-ceph-rgw-biohazard-a
namespace: rook-ceph
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization

42
kube/deploy/core/storage/minio-nas/app/netpol.yaml Normal file
View File

@@ -0,0 +1,42 @@
---
# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: &app minio-nas
namespace: *app
spec:
endpointSelector: {}
ingress:
# same namespace
- fromEndpoints:
- matchLabels:
io.kubernetes.pod.namespace: *app
# allow HTTPS traffic in-cluster
- fromEndpoints:
- matchLabels:
s3.home.arpa/minio-nas: allow
matchExpressions:
- key: io.kubernetes.pod.namespace
operator: Exists
toPorts:
- ports:
- port: "9000"
---
# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumclusterwidenetworkpolicy_v2.json
apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
name: &app minio-nas-in-cluster
spec:
endpointSelector:
matchLabels:
s3.home.arpa/minio-nas: allow
egress:
- toEndpoints:
- matchLabels:
io.kubernetes.pod.namespace: minio-nas
app.kubernetes.io/name: minio-nas
toPorts:
- ports:
- port: "9000"

7
kube/deploy/core/storage/volsync/app/netpol.yaml
View File

@@ -35,6 +35,10 @@ spec:
endpointSelector:
matchLabels:
app.kubernetes.io/created-by: volsync
ingress:
- fromEndpoints:
- matchLabels:
prometheus: "kps"
egress:
# allow all to public Internet
- toEntities:
@@ -51,3 +55,6 @@ spec:
toPorts:
- ports:
- port: "6953"
- toEndpoints:
- matchLabels:
rgw: "${CLUSTER_NAME}"