diff --git a/.renovate/preUpdateBackups.json5 b/.renovate/preUpdateBackups.json5 new file mode 100644 index 00000000..2c982b70 --- /dev/null +++ b/.renovate/preUpdateBackups.json5 @@ -0,0 +1,16 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "customManagers": [ + { + "customType": "regex", + "description": "Process Docker packages for pre-update VolSync/PG backups in ks.yaml", + "managerFilePatterns": ["/^kube/.+/ks\\.ya?ml$/"], + "matchStrings": [ + // VS_APP_CURRENT_VERSION: "{{.IMAGENAME}}:{{.IMAGETAG}}" + " *(?VS|PG)_APP_CURRENT_VERSION: \"?(?[\\w\\d\\-\\_\\/\\.]+):(?[\\w\\d\\.\\-\\_]+)\"?" + ], + "datasourceTemplate": "docker", + "versioningTemplate": "docker" + } + ] +} diff --git a/.renovaterc.json5 b/.renovaterc.json5 index 6b61ee7c..35ec10de 100644 --- a/.renovaterc.json5 +++ b/.renovaterc.json5 @@ -17,6 +17,7 @@ //"github>JJGadgets/Biohazard//.renovate/grafanaDashboards.json5", "github>buroa/k8s-gitops//.renovate/grafanaDashboards.json5#1b97193da1d11d80918f02de13840aa9f28ef06f", "github>JJGadgets/Biohazard//.renovate/mise.json5", + "github>JJGadgets/Biohazard//.renovate/preUpdateBackups.json5", "github>JJGadgets/Biohazard//.renovate/groups.json5", "github>JJGadgets/Biohazard//.renovate/clusters.json5", "github>JJGadgets/Biohazard//.renovate/commitMessage.json5", diff --git a/.taskfiles/k8s/Taskfile.dist.yaml b/.taskfiles/k8s/Taskfile.dist.yaml index 9ac76139..75366bd1 100644 --- a/.taskfiles/k8s/Taskfile.dist.yaml +++ b/.taskfiles/k8s/Taskfile.dist.yaml @@ -89,7 +89,7 @@ tasks: vars: dns: '{{ or .dns (fail "Missing search query!") }}' cmds: - - kubectl get ingress,svc -A -o yaml | yq -r '.items | map(select(.metadata.annotations."external-dns.alpha.kubernetes.io/target"=="*{{.dns}}*").metadata.name)' + - kubectl get ingress,svc --all-namespaces=true -o yaml | yq -r '.items | map(select(.metadata.annotations."external-dns.alpha.kubernetes.io/target"=="*{{.dns}}*").metadata.name)' sops-apply: aliases: [sa] @@ -126,10 +126,14 @@ tasks: desc: Copy app folder structure template, substitute APPNAME, and (TODO) prompt user for variables values such as DNS, UID etc. vars: APP: &app-fail '{{ or .APP (fail "Missing `app` variable!") }}' + IMAGENAME: &imagename-fail '{{ or .IMAGENAME (fail "Missing `IMAGENAME` variable!") }}' + IMAGETAG: &imagetag-fail '{{ or .IMAGETAG (fail "Missing `IMAGETAG` variable!") }}' cmds: - cp -r ./kube/templates/test ./kube/deploy/apps/{{.APP}} # lowercase, used for resource names etc - grep -lR 'APPNAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/${APPNAME}/{{.APP}}/g' % + - grep -lR 'IMAGENAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/${IMAGENAME}/{{.IMAGENAME}}/g' % + - grep -lR 'IMAGETAG' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/${IMAGETAG}/{{.IMAGETAG}}/g' % # uppercase, for variable substitution references e.g. ${APP_DNS_AUTHENTIK} - grep -lR 'APPNAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/_APPNAME:=/_{{.APP}}:=/g;s/\(_{{.APP}}:=\)/\U\1/g' % - grep -lR 'APPNAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/_APPNAME}/_{{.APP}}}/g;s/\(_{{.APP}}}\)/\U\1/g' % @@ -146,9 +150,9 @@ tasks: cmds: - |- while true; do - kubectl delete pod -A --field-selector=status.phase==Failed || true; - kubectl delete pod -A --field-selector=status.phase==Pending || true; - kubectl delete pod -A --field-selector=status.phase==Succeeded || true; + kubectl delete pod --wait=false --all-namespaces=true --field-selector=status.phase==Failed || true; + kubectl delete pod --wait=false --all-namespaces=true --field-selector=status.phase==Pending || true; + kubectl delete pod --wait=false --all-namespaces=true --field-selector=status.phase==Succeeded || true; done delete-stuck-pvc: diff --git a/kube/deploy/core/db/pg/clusters/template/crunchy.yaml b/kube/deploy/core/db/pg/clusters/template/crunchy.yaml index c387515f..4d10ddaa 100644 --- a/kube/deploy/core/db/pg/clusters/template/crunchy.yaml +++ b/kube/deploy/core/db/pg/clusters/template/crunchy.yaml @@ -72,9 +72,9 @@ spec: name: "pg-${PG_APP_NAME}-secrets" - secret: name: "pg-${PG_APP_NAME}-s3-crunchy" - #manual: - # repoName: "repo2" - # options: ["--type=full", "--annotation=reason=change-R2-buckets", "--archive-copy", "--checksum-page"] + manual: + repoName: "repo2" + options: ["--type=full", "--annotation=reason=${PG_APP_CURRENT_VERSION:=manual}", "--archive-copy", "--checksum-page"] global: &brflag archive-timeout: "60" compress-type: "bz2" diff --git a/kube/deploy/core/storage/volsync/template/rsrc-r2.yaml b/kube/deploy/core/storage/volsync/template/rsrc-r2.yaml index 4a3f879c..b2a13eea 100644 --- a/kube/deploy/core/storage/volsync/template/rsrc-r2.yaml +++ b/kube/deploy/core/storage/volsync/template/rsrc-r2.yaml @@ -46,51 +46,51 @@ spec: matchExpressions: - key: kubernetes.io/metadata.name operator: Exists -# --- -# # yaml-language-server: $schema=https://crds.jank.ing/volsync.backube/replicationsource_v1alpha1.json -# apiVersion: volsync.backube/v1alpha1 -# kind: ReplicationSource -# metadata: -# name: "${PVC}-r2-updates-restic" -# labels: -# volsync.home.arpa/repo: "r2" -# spec: -# sourcePVC: "${PVC}" -# trigger: -# manual: "${VS_APP_CURRENT_VERSION:=manual}" -# restic: -# copyMethod: "Snapshot" -# repository: "vs-${PVC}-r2-restic" -# volumeSnapshotClassName: "${SNAP:=file}" -# storageClassName: "${SC:=file}" -# accessModes: ["${SNAP_ACCESSMODE:=ReadOnlyMany}"] # CephFS shallow snapshot, and allow rsrc pod to schedule anywhere -# cacheStorageClassName: "${CACHE:=block}" -# cacheAccessModes: ["ReadWriteOnce"] -# cacheCapacity: "${CACHESIZE:=2Gi}" -# moverSecurityContext: -# runAsNonRoot: true -# runAsUser: ${RUID:=1000} -# runAsGroup: ${RGID:=1000} -# fsGroup: ${RFSG:=1000} -# seccompProfile: {type: "RuntimeDefault"} -# pruneIntervalDays: ${PRUNE:=14} -# retain: # keep all backups within 1 week (7 days), keep latest snapshot from each day within 2 weeks -# daily: ${DAILY:=14} -# within: ${WITHIN:=7d} -# moverPodLabels: -# egress.home.arpa/r2: allow -# moverResources: -# limits: -# cpu: 500m -# memory: 2Gi -# moverAffinity: -# podAntiAffinity: -# requiredDuringSchedulingIgnoredDuringExecution: -# - topologyKey: kubernetes.io/hostname -# labelSelector: -# matchLabels: -# app.kubernetes.io/created-by: volsync -# namespaceSelector: -# matchExpressions: -# - key: kubernetes.io/metadata.name -# operator: Exists +--- +# yaml-language-server: $schema=https://crds.jank.ing/volsync.backube/replicationsource_v1alpha1.json +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationSource +metadata: + name: "${PVC}-r2-updates-restic" + labels: + volsync.home.arpa/repo: "r2" +spec: + sourcePVC: "${PVC}" + trigger: + manual: "${VS_APP_CURRENT_VERSION:=manual}" + restic: + copyMethod: "Snapshot" + repository: "vs-${PVC}-r2-restic" + volumeSnapshotClassName: "${SNAP:=file}" + storageClassName: "${SC:=file}" + accessModes: ["${SNAP_ACCESSMODE:=ReadOnlyMany}"] # CephFS shallow snapshot, and allow rsrc pod to schedule anywhere + cacheStorageClassName: "${CACHE:=block}" + cacheAccessModes: ["ReadWriteOnce"] + cacheCapacity: "${CACHESIZE:=2Gi}" + moverSecurityContext: + runAsNonRoot: true + runAsUser: ${RUID:=1000} + runAsGroup: ${RGID:=1000} + fsGroup: ${RFSG:=1000} + seccompProfile: {type: "RuntimeDefault"} + pruneIntervalDays: ${PRUNE:=14} + retain: # keep all backups within 1 week (7 days), keep latest snapshot from each day within 2 weeks + daily: ${DAILY:=14} + within: ${WITHIN:=7d} + moverPodLabels: + egress.home.arpa/r2: allow + moverResources: + limits: + cpu: 500m + memory: 2Gi + moverAffinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/created-by: volsync + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: Exists diff --git a/kube/templates/test/app/hr.yaml b/kube/templates/test/app/hr.yaml index dc97753a..5fa7ef94 100644 --- a/kube/templates/test/app/hr.yaml +++ b/kube/templates/test/app/hr.yaml @@ -1,5 +1,5 @@ --- -# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/app-template-3.6.1/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/app-template-4.0.1/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -10,14 +10,14 @@ spec: chart: spec: chart: app-template - version: 3.7.3 + version: 4.0.1 sourceRef: name: bjw-s kind: HelmRepository namespace: flux-system values: controllers: - ${APPNAME}: + app: # type: statefulset type: deployment replicas: 1 @@ -30,10 +30,10 @@ spec: # prom.home.arpa/kps: allow # egress.home.arpa/internet: allow containers: - main: + app: image: &img - repository: ghcr.io/${APPNAME}/${APPNAME} - tag: v + repository: ${IMAGENAME} + tag: ${IMAGETAG} env: &env TZ: "${CONFIG_TZ}" _APPNAME_DATABASE_SOURCE: @@ -70,8 +70,12 @@ spec: enabled: true readiness: enabled: true - #startup: - # enabled: true + # startup: + # enabled: true + # spec: + # periodSeconds: 1 + # failureThreshold: 300 + # initialDelaySeconds: 5 # statefulset: # volumeClaimTemplates: # - name: data @@ -89,33 +93,34 @@ spec: # storageClass: block # globalMounts: # - path: /backup - initContainers: - 01-init-${APPNAME}-admin-password: - image: *img - command: - - /bin/sh - - -c - - "[ -s /data/${APPNAME}.db ] || /sbin/${APPNAME}d recover_account -c /data/server.toml admin" - securityContext: *sc - #resources: - 01-init-db: - image: - repository: "ghcr.io/onedr0p/postgres-init" - tag: "17.4@sha256:43dd04e91e861cf912378bad987afa168fa4f13d05528304907ad0aa351195d6" - envFrom: [secretRef: { name: "${APPNAME}-pg-superuser" }] - securityContext: *sc - #resources: + # initContainers: + # 01-init-${APPNAME}-admin-password: + # image: *img + # command: + # - /bin/sh + # - -c + # - "[ -s /data/${APPNAME}.db ] || /sbin/${APPNAME}d recover_account -c /data/server.toml admin" + # securityContext: *sc + # #resources: + # 01-init-db: + # image: + # repository: "ghcr.io/onedr0p/postgres-init" + # tag: "17.4@sha256:43dd04e91e861cf912378bad987afa168fa4f13d05528304907ad0aa351195d6" + # envFrom: [secretRef: { name: "${APPNAME}-pg-superuser" }] + # securityContext: *sc + # #resources: service: - ${APPNAME}: - controller: ${APPNAME} + app: + controller: app ports: http: - port: 8080 + port: 80 + targetPort: 8080 protocol: HTTP appProtocol: http expose: primary: false - controller: ${APPNAME} + controller: app type: LoadBalancer annotations: coredns.io/hostname: "${APP_DNS_APPNAME:=${APPNAME}}" @@ -137,9 +142,9 @@ spec: main: className: nginx-internal annotations: + nginx.ingress.kubernetes.io/whitelist-source-range: "${IP_JJ_V4:=127.0.0.1/32}" external-dns.alpha.kubernetes.io/target: "${DNS_CF:=cf}" external-dns.alpha.kubernetes.io/cloudflare-proxied: "true" - nginx.ingress.kubernetes.io/whitelist-source-range: "${IP_JJ_V4:=127.0.0.1/32}" nginx.ingress.kubernetes.io/backend-protocol: HTTPS # https://github.com/kubernetes/ingress-nginx/issues/6728 nginx.ingress.kubernetes.io/server-snippet: | @@ -154,14 +159,14 @@ spec: - path: / pathType: Prefix service: - identifier: ${APPNAME} + identifier: app port: http tls: - hosts: [*host] persistence: config: type: configMap - name: ${APPNAME}-config + identifier: config advancedMounts: ${APPNAME}: main: @@ -182,6 +187,7 @@ spec: tmp: type: emptyDir medium: Memory + sizeLimit: 16Mi globalMounts: - subPath: tmp path: /tmp @@ -200,9 +206,8 @@ spec: readOnly: true configMaps: config: - enabled: true data: - server.toml: |- + server.toml: | domain = "${APP_DNS_APPNAME}" origin = "https://${APP_DNS_APPNAME}" tls_chain = "/tls/fullchain.pem" @@ -246,18 +251,18 @@ spec: whenUnsatisfiable: DoNotSchedule labelSelector: matchLabels: - app.kubernetes.io/name: *app + app.kubernetes.io/name: "{{ .Release.Name }}" affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: fuckoff.home.arpa/${APPNAME} + - key: "fuckoff.home.arpa/{{ .Release.Name }}" operator: DoesNotExist networkpolicies: same-ns: # either - controller: ${APPNAME} + controller: app # or podSelector: {} # end diff --git a/kube/templates/test/ks.yaml b/kube/templates/test/ks.yaml index 449f07dd..cfa93fc9 100644 --- a/kube/templates/test/ks.yaml +++ b/kube/templates/test/ks.yaml @@ -12,8 +12,28 @@ spec: labels: *l path: ./kube/deploy/apps/${APPNAME}/app components: - - ../../../core/storage/volsync/component/ - ../../../core/flux-system/alerts/template/ + dependsOn: + - name: crds + namespace: flux-system + - name: ${APPNAME}-db + - name: ${APPNAME}-pvc +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: ${APPNAME}-pvc + namespace: flux-system + labels: &l + app.kubernetes.io/name: ${APPNAME} + pvc.home.arpa/volsync: "true" +spec: + targetNamespace: ${APPNAME} + commonMetadata: + labels: *l + path: ./kube/deploy/core/storage/volsync/template + components: + - ../../../core/storage/volsync/component/ dependsOn: - name: crds namespace: flux-system @@ -34,6 +54,7 @@ spec: # ${APP_UID_APPNAME:=1000} # RFSG: !!str | # ${APP_UID_APPNAME:=1000} + VS_APP_CURRENT_VERSION: "{{.IMAGENAME}}:{{.IMAGETAG}}" --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization @@ -57,3 +78,4 @@ spec: PG_NAME: "default" PG_DB_USER: &app "${APPNAME}" PG_APP_NS: *app + PG_APP_CURRENT_VERSION: "{{.IMAGENAME}}:{{.IMAGETAG}}" diff --git a/kube/templates/test/kustomization.yaml b/kube/templates/test/kustomization.yaml index 45bc3673..5eeb2657 100644 --- a/kube/templates/test/kustomization.yaml +++ b/kube/templates/test/kustomization.yaml @@ -2,5 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - # - ns.yaml + - ns.yaml - ks.yaml diff --git a/kube/templates/test/app/ns.yaml b/kube/templates/test/ns.yaml similarity index 100% rename from kube/templates/test/app/ns.yaml rename to kube/templates/test/ns.yaml