feat(soft-serve): app-template v2, use CephFS PVC

kube/deploy/apps/soft-serve/app/es.yaml

@@ -0,0 +1,19 @@
+---
+# yaml-language-server: $schema=https://crds.jank.ing/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: &name soft-serve-secrets
+  namespace: soft-serve
+spec:
+  refreshInterval: 1m
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: 1p
+  dataFrom:
+    - extract:
+        key: "soft-serve - ${CLUSTER_NAME}"
+  target:
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    name: *name

kube/deploy/apps/soft-serve/app/hr.yaml

@@ -5,64 +5,77 @@ metadata:
   name: &app soft-serve
   namespace: *app
 spec:
+  interval: 5m
   chart:
     spec:
       chart: app-template
-      version: 1.5.1
+      version: "2.6.0"
       sourceRef:
         name: bjw-s
         kind: HelmRepository
         namespace: flux-system
   values:
-    global:
-      fullnameOverride: *app
-    automountServiceAccountToken: false
-    controller:
-      type: deployment
-      replicas: 1
-    image:
-      repository: ghcr.io/charmbracelet/soft-serve
-      tag: v0.7.4@sha256:1e89075219cb95f62e912fb1831b170bcec73a30e2bb6431533e12dfca2666b0
-    podLabels:
-      ingress.home.arpa/nginx-internal: "allow"
-      db.home.arpa/pg: "pg-default"
-    env:
-      TZ: "${CONFIG_TZ}"
-      SOFT_SERVE_NAME: "BioFlurry"
-      SOFT_SERVE_LOG_FORMAT: "json"
-      SOFT_SERVE_SSH_LISTEN_ADDR: ":23231"
-      SOFT_SERVE_SSH_PUBLIC_URL: "ssh://${APP_DNS_SOFT_SERVE}:22"
-      SOFT_SERVE_SSH_KEY_PATH: "/ssh/host-priv"
-      SOFT_SERVE_SSH_CLIENT_KEY_PATH: "/ssh/client-priv"
-      SOFT_SERVE_GIT_LISTEN_ADDR: ":9418"
-      SOFT_SERVE_HTTP_LISTEN_ADDR: ":23232"
-      SOFT_SERVE_HTTP_PUBLIC_URL: "https://${APP_DNS_SOFT_SERVE_HTTPS}"
-      SOFT_SERVE_STATS_LISTEN_ADDR: ":23233"
-      SOFT_SERVE_DB_DRIVER: "postgres"
-      SOFT_SERVE_DB_DATA_SOURCE: {valueFrom: {secretKeyRef: {name: pg-default-pguser-softserve, key: pgbouncer-uri}}}
-      SOFT_SERVE_INITIAL_ADMIN_KEYS: |
-        ${SECRET_ADMIN_SSH_PUBKEY_1}
-        ${SECRET_ADMIN_SSH_PUBKEY_2}
-        ${SECRET_ADMIN_SSH_PUBKEY_3}
-        ${SECRET_ADMIN_SSH_PUBKEY_4}
+    controllers:
+      main:
+        type: deployment
+        replicas: 1
+        pod:
+          labels:
+            db.home.arpa/pg: "pg-default"
+            ingress.home.arpa/world: "allow"
+            ingress.home.arpa/cluster: "allow"
+        containers:
+          main:
+            image: &img
+              repository: "ghcr.io/charmbracelet/soft-serve"
+              tag: "v0.7.4@sha256:1e89075219cb95f62e912fb1831b170bcec73a30e2bb6431533e12dfca2666b0"
+            env:
+              TZ: "${CONFIG_TZ}"
+              SOFT_SERVE_NAME: "BioFlurry"
+              SOFT_SERVE_LOG_FORMAT: "json"
+              SOFT_SERVE_SSH_LISTEN_ADDR: ":23231"
+              SOFT_SERVE_SSH_PUBLIC_URL: "ssh://${APP_DNS_SOFT_SERVE}:22"
+              SOFT_SERVE_SSH_KEY_PATH: "/secrets/host-priv"
+              SOFT_SERVE_SSH_CLIENT_KEY_PATH: "/secrets/client-priv"
+              SOFT_SERVE_GIT_LISTEN_ADDR: ":9418"
+              SOFT_SERVE_HTTP_LISTEN_ADDR: ":23232"
+              SOFT_SERVE_HTTP_PUBLIC_URL: "https://${APP_DNS_SOFT_SERVE}"
+              SOFT_SERVE_HTTP_TLS_CERT_PATH: "/tls/tls.crt"
+              SOFT_SERVE_HTTP_TLS_KEY_PATH: "/tls/tls.key"
+              SOFT_SERVE_STATS_LISTEN_ADDR: ":23233"
+              SOFT_SERVE_LFS_ENABLED: "true"
+              SOFT_SERVE_LFS_SSH_ENABLED: "true"
+              SOFT_SERVE_DB_DRIVER: "postgres"
+              SOFT_SERVE_DB_DATA_SOURCE: {valueFrom: {secretKeyRef: {name: pg-default-pguser-softserve, key: pgbouncer-uri}}}
+              SOFT_SERVE_INITIAL_ADMIN_KEYS: {valueFrom: {secretKeyRef: {name: soft-serve-secrets, key: admin-keys}}}
+            securityContext: &sc
+              readOnlyRootFilesystem: true
+              allowPrivilegeEscalation: false
+              capabilities:
+                drop: ["ALL"]
+            resources:
+              requests:
+                cpu: "10m"
+                memory: "128Mi"
+              limits:
+                cpu: "3000m"
+                memory: "1Gi"
     service:
       main:
         enabled: true
         primary: true
-        type: ClusterIP
-        ports:
-          http:
-            port: 23232
-          stats:
-            port: 23233
-      expose:
-        enabled: true
+        controller: main
         type: LoadBalancer
         externalTrafficPolicy: Cluster
         annotations:
           coredns.io/hostname: "${APP_DNS_SOFT_SERVE}"
           "io.cilium/lb-ipam-ips": "${APP_IP_SOFT_SERVE}"
         ports:
+          http:
+            enabled: true
+            port: 443
+            targetPort: 23232
+            protocol: HTTPS
           ssh:
             enabled: true
             port: 22
@@ -73,41 +86,50 @@ spec:
             port: 9418
             targetPort: 9418
             protocol: TCP
-    ingress:
-      main:
-        enabled: true
-        primary: true
-        ingressClassName: "nginx-internal"
-        hosts:
-          - host: &host "${APP_DNS_SOFT_SERVE_HTTPS}"
-            paths:
-              - path: /
-                pathType: Prefix
-        tls:
-          - hosts:
-              - *host
-    podSecurityContext:
-      runAsUser: &uid ${APP_UID_SOFT_SERVE}
-      runAsGroup: *uid
-      fsGroup: *uid
-      fsGroupChangePolicy: Always
+          stats:
+            port: 23233
     persistence:
-      data:
-        enabled: true
-        type: nfs
-        server: "${IP_TRUENAS}"
-        path: "${PATH_NAS_PERSIST_K8S}/soft-serve"
-        mountPath: "/soft-serve"
-      ssh:
+      config:
         enabled: true
         type: secret
-        name: soft-serve-ssh
-        mountPath: /ssh
-        readOnly: true
-    resources:
-      requests:
-        cpu: 10m
-        memory: 128Mi
-      limits:
-        cpu: 3000m
-        memory: 6000Mi
+        name: "soft-serve-secrets"
+        globalMounts:
+          - path: "/secrets"
+      data:
+        enabled: true
+        existingClaim: "soft-serve-data"
+        globalMounts:
+          - subPath: "data"
+            path: "/soft-serve"
+      tls:
+        enabled: true
+        type: secret
+        name: "short-domain-tls"
+        defaultMode: 0400
+        globalMounts:
+          - path: "/tls"
+            readOnly: true
+    defaultPodOptions:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: &uid ${APP_UID_SOFT_SERVE}
+        runAsGroup: *uid
+        fsGroup: *uid
+        fsGroupChangePolicy: "Always"
+        seccompProfile: { type: "RuntimeDefault" }
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: "kubernetes.io/hostname"
+          whenUnsatisfiable: "DoNotSchedule"
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: *app
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: "fuckoff.home.arpa/soft-serve"
+                    operator: "DoesNotExist"

kube/deploy/apps/soft-serve/app/netpol.yaml

@@ -1,31 +0,0 @@
----
-# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: &app soft-serve
-  namespace: *app
-spec:
-  endpointSelector: {}
-  ingress:
-    # same namespace
-    - fromEndpoints:
-        - matchLabels:
-            io.kubernetes.pod.namespace: *app
-    - fromEntities:
-        - "world"
-      toPorts:
-        - ports:
-            - port: "22"
-              protocol: TCP
-        - ports:
-            - port: "23231"
-              protocol: TCP
-        - ports:
-            - port: "9418"
-              protocol: TCP
-  egress:
-    # same namespace
-    - toEndpoints:
-        - matchLabels:
-            io.kubernetes.pod.namespace: *app

kube/deploy/apps/soft-serve/app/ssh.yaml

@@ -1,40 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: soft-serve-ssh
-    namespace: soft-serve
-type: Opaque
-stringData:
-    host-priv: ENC[AES256_GCM,data:mKJd9FH2Z9GyxSS0mTdiauImrobI+kqZ013ovtSNW46rEI59OAHuVwv/k/82FzTvCg/Aju4tqTfPhP6/Ivf/q0G1WRdAbRKOGdJ0picJ9Z3ETv6QQ1FVSkHCHmvW8BBG2BrHk80RcS3bx6VQa2v2D9KnFOEtdmUn5Afi6v8232B/ShvfX2QnSVuEH2WCiDJpZYp/vPRDC5deJqR1kYrlO0w/dSdo73LD/jU3XMxlOwLTaYnUIt+f8upyOY7e7x4EIKnE3E9DslG5LEdlG9hXDgGi+s6droYhAd70FGDrFTwguu3slMutt6Jr4nS4HfYsnBnkdhFmAqosWOYzMAarPxFbaPPVYJJK+acGn+iABCVXHE3bOx2Tr6MWAaSUvBi5n9SvdnI/CypzCfNiuDfWlrnv00OJLzgZSsmBSApUaA3oYkOYwJ6pelmajYMY3+wbIRWkxPFimU0Z+AcjVrOYkcub35UNPgzkYoB3uU4HmXjlVFWaIngURWQMe6AlPm6uPRZvh+2vz1xjBhC6rQN7pW2gMM0eq7xL8ZE3Riqbl/0s6sXba1XGWVfIKWXpYQsAJAc7xep5o+vburBB,iv:XahBh/d3u7DCc2jWvDufio9yqkxBRd0NOKJCmzYlrOI=,tag:4//267lH80qX/KakAHgZVw==,type:str]
-    client-priv: ENC[AES256_GCM,data:A/zmVmVXajbRkEsM7YHCAwF3bqBGGFhYH5eDF/KdnSyIy7Vd8Aa1txjpymLqNFbIwoHbvRZT4+bHVpD0Cd8jqSaQ6EnHv9jPQokLXxi/L5iyboCfILYGNyAa+guojK8sekJhGl5pSpQdhiwcff7JejbZNJXsuHVysZthnFobx72dh07zJ8rYO2ARBs530VLe5fcCH1RmmWj902B28RT3mMI8ZsesIzx41LucWM+z6n8AskifD15hrobImkLeIVNbaKnDxq04BDznCeu/t9wRKEw7hgMrzh0CREzWaIDv/GlSbpIQjhIdmUr9sgsMEs0gEom2pNy64yfpizEQP6JiOSmQ41wnqF3MR4AwM/B6mHkfbhJVt1B1O3zU2xXe7qN9I/k6SgopCBJtB621Ytzx4cZWZDwjdVfbUUWUChVxvvwV7wNTbGa7M9BaKENrVvvYHqC35Wlb+SlsxpQN+jjApYq85iS7DW0qSLk39JUCG+ZZ7ZTIw7X7x4ABLJ+ZHaQiZRMTH95rTqzqUHFWTMHWA1HeRkwSQklBGJHFO/fFOot5k0PVkuiSJRLvHo5//Pk6qn124uOPlhKNeuA=,iv:kUWc+NoR0GJ5AjqEJsjiEl1upWfwcbvFF+Itel9zdvQ=,tag:dudsRIBvkuVHbm0mqUCRnA==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1u57l4s400gqstc0p485j4646cemntufr0pcyp32yudklsp90xpmszxvnkj
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwdkhXVUlzZkVoUkhSWjM2
-            b055TWlUSTFlbUZaQzFyWnJnYVNKVlFzVEZ3Cnc2eHQ5N0ZHMGx1eXduSnljcmg3
-            NmIwWjdyY1ljZVErNVJyTUZpTWJaa28KLS0tIDlkL3Q2VDZWeDBqMmphK3dzOWJI
-            dlJ2TWpPSTd6UExmWDdCZlNkNW5IYWcKXJpSqhh7Os21uI81t2t0LXOgrahJ85JK
-            +S3tdCFxmt8dsgNT/X0fjl5klGFp6KAd9VZI1D0MdF99kkZnwqGH/A==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-08-07T02:23:25Z"
-    mac: ENC[AES256_GCM,data:IgrrnGr0ixfYYlHRzSIRgBQneMHF+NFHB3rDYntcvvcIFYreNqAnGzEemhJ2efC8FucNifi1WOS4LFZmTN/yWTUoW2GOIcTL/1+AsnVw9NHvXqMOWr5Ltd4SJCHB7zYjhzsMtUW5+8EcqAgqod5S3MrBeTfQOxulqudPDIfsE0k=,iv:Oc9Tb+I1fKNdOOU07eBNKpRZiG4a76y7xydEdeA5aPs=,tag:hwnbxnv7JEHd0u2seYyk0A==,type:str]
-    pgp:
-        - created_at: "2023-08-07T02:19:08Z"
-          enc: |
-            -----BEGIN PGP MESSAGE-----
-
-            hF4DAAAAAAAAAAASAQdACBYfXdyXxsT7K4N5TzgLabHY7oYm0lyps1HahU4XzEIw
-            QmMFwMUkpDPJNmM/uYG5zASXrnk5y53boDJdNObUdxH+8tx5fUlCDVfK6Rhx2I+H
-            0l4B2H5UlRyDCONzp4N49SDUv2Gd33ZOpZDDy+B3x9GZ6SYSVZ8RNRLCSFYzP5C8
-            3P4lzFQzAkeLKCCmT7CPHgePcui91cOJ/EMc/gzGWYZBdIXXCyzXESZSW2y4wm/6
-            =S81P
-            -----END PGP MESSAGE-----
-          fp: 31E70E5BC80C58AFF5DD649921AC5A1AC6E5B7F2
-    encrypted_regex: ^(data|stringData)$
-    version: 3.7.3

kube/deploy/apps/soft-serve/ks.yaml

@@ -4,23 +4,58 @@ kind: Kustomization
 metadata:
   name: soft-serve-app
   namespace: flux-system
-  labels:
-    prune.flux.home.arpa/enabled: "true"
+  labels: &l
+    app.kubernetes.io/name: "soft-serve"
 spec:
-  prune: true
+  commonMetadata:
+    labels: *l
   path: ./kube/deploy/apps/soft-serve/app
+  targetNamespace: "soft-serve"
   dependsOn:
     - name: soft-serve-db
+    - name: soft-serve-pvc
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: soft-serve-pvc
+  namespace: flux-system
+  labels: &l
+    app.kubernetes.io/name: "soft-serve"
+spec:
+  commonMetadata:
+    labels: *l
+  path: ./kube/deploy/core/storage/volsync/template
+  targetNamespace: "soft-serve"
+  dependsOn:
+    - name: 1-core-storage-volsync-app
+    - name: 1-core-storage-rook-ceph-cluster
+  postBuild:
+    substitute:
+      PVC: "soft-serve-data"
+      SIZE: "20Gi"
+      SC: &sc "file"
+      SNAP: *sc
+      ACCESSMODE: "ReadWriteMany"
+      RUID: !!str &uid |
+        ${APP_UID_SOFT_SERVE}
+      RGID: !!str |
+        ${APP_UID_SOFT_SERVE}
+      RFSG: !!str |
+        ${APP_UID_SOFT_SERVE}
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: soft-serve-db
   namespace: flux-system
-  labels:
+  labels: &l
     prune.flux.home.arpa/enabled: "true"
     db.home.arpa/pg: "pg-default"
+    app.kubernetes.io/name: "soft-serve"
 spec:
+  commonMetadata:
+    labels: *l
   path: ./kube/deploy/core/db/pg/clusters/template/pguser
   targetNamespace: "pg"
   dependsOn:

kube/deploy/apps/soft-serve/ns.yaml

@@ -3,3 +3,8 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: soft-serve
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    pod-security.kubernetes.io/enforce: &ps restricted
+    pod-security.kubernetes.io/audit: *ps
+    pod-security.kubernetes.io/warn: *ps

kube/deploy/apps/soft-serve/volsync.yaml

@@ -1,36 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: soft-serve-restic
-  namespace: soft-serve
-type: Opaque
-stringData:
-  RESTIC_REPOSITORY: ${SECRET_VOLSYNC_R2_REPO}/soft-serve
-  RESTIC_PASSWORD: ${SECRET_VOLSYNC_PASSWORD}
-  AWS_ACCESS_KEY_ID: ${SECRET_VOLSYNC_R2_ID}
-  AWS_SECRET_ACCESS_KEY: ${SECRET_VOLSYNC_R2_KEY}
----
-apiVersion: volsync.backube/v1alpha1
-kind: ReplicationSource
-metadata:
-  name: soft-serve-restic
-  namespace: soft-serve
-spec:
-  sourcePVC: config-soft-serve-0 # TODO: find PVC name
-  trigger:
-    schedule: "0 22 * * *" # 6am GMT+8
-  restic:
-    copyMethod: Snapshot
-    pruneIntervalDays: 14
-    repository: soft-serve-restic
-    cacheCapacity: 2Gi
-    volumeSnapshotClassName: block
-    storageClassName: block
-    moverSecurityContext:
-      runAsUser: &uid ${APP_UID_SOFT_SERVE}
-      runAsGroup: ${APP_UID_SOFT_SERVE}
-      fsGroup: ${APP_UID_SOFT_SERVE}
-    retain:
-      daily: 14
-      within: 7d