From 09ef6ba27ff79cfb2411a67ecd84b816a53e9c32 Mon Sep 17 00:00:00 2001 From: JJGadgets Date: Mon, 25 Mar 2024 11:52:26 +0800 Subject: [PATCH] feat(soft-serve): app-template v2, use CephFS PVC --- kube/deploy/apps/soft-serve/app/es.yaml | 19 +++ kube/deploy/apps/soft-serve/app/hr.yaml | 172 +++++++++++--------- kube/deploy/apps/soft-serve/app/netpol.yaml | 31 ---- kube/deploy/apps/soft-serve/app/ssh.yaml | 40 ----- kube/deploy/apps/soft-serve/ks.yaml | 43 ++++- kube/deploy/apps/soft-serve/ns.yaml | 5 + kube/deploy/apps/soft-serve/volsync.yaml | 36 ---- 7 files changed, 160 insertions(+), 186 deletions(-) create mode 100644 kube/deploy/apps/soft-serve/app/es.yaml delete mode 100644 kube/deploy/apps/soft-serve/app/netpol.yaml delete mode 100644 kube/deploy/apps/soft-serve/app/ssh.yaml delete mode 100644 kube/deploy/apps/soft-serve/volsync.yaml diff --git a/kube/deploy/apps/soft-serve/app/es.yaml b/kube/deploy/apps/soft-serve/app/es.yaml new file mode 100644 index 00000000..c6a4c0d9 --- /dev/null +++ b/kube/deploy/apps/soft-serve/app/es.yaml @@ -0,0 +1,19 @@ +--- +# yaml-language-server: $schema=https://crds.jank.ing/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: &name soft-serve-secrets + namespace: soft-serve +spec: + refreshInterval: 1m + secretStoreRef: + kind: ClusterSecretStore + name: 1p + dataFrom: + - extract: + key: "soft-serve - ${CLUSTER_NAME}" + target: + creationPolicy: Owner + deletionPolicy: Retain + name: *name diff --git a/kube/deploy/apps/soft-serve/app/hr.yaml b/kube/deploy/apps/soft-serve/app/hr.yaml index 1b682921..985773c7 100644 --- a/kube/deploy/apps/soft-serve/app/hr.yaml +++ b/kube/deploy/apps/soft-serve/app/hr.yaml @@ -5,64 +5,77 @@ metadata: name: &app soft-serve namespace: *app spec: + interval: 5m chart: spec: chart: app-template - version: 1.5.1 + version: "2.6.0" sourceRef: name: bjw-s kind: HelmRepository namespace: flux-system values: - global: - fullnameOverride: *app - automountServiceAccountToken: false - controller: - type: deployment - replicas: 1 - image: - repository: ghcr.io/charmbracelet/soft-serve - tag: v0.7.4@sha256:1e89075219cb95f62e912fb1831b170bcec73a30e2bb6431533e12dfca2666b0 - podLabels: - ingress.home.arpa/nginx-internal: "allow" - db.home.arpa/pg: "pg-default" - env: - TZ: "${CONFIG_TZ}" - SOFT_SERVE_NAME: "BioFlurry" - SOFT_SERVE_LOG_FORMAT: "json" - SOFT_SERVE_SSH_LISTEN_ADDR: ":23231" - SOFT_SERVE_SSH_PUBLIC_URL: "ssh://${APP_DNS_SOFT_SERVE}:22" - SOFT_SERVE_SSH_KEY_PATH: "/ssh/host-priv" - SOFT_SERVE_SSH_CLIENT_KEY_PATH: "/ssh/client-priv" - SOFT_SERVE_GIT_LISTEN_ADDR: ":9418" - SOFT_SERVE_HTTP_LISTEN_ADDR: ":23232" - SOFT_SERVE_HTTP_PUBLIC_URL: "https://${APP_DNS_SOFT_SERVE_HTTPS}" - SOFT_SERVE_STATS_LISTEN_ADDR: ":23233" - SOFT_SERVE_DB_DRIVER: "postgres" - SOFT_SERVE_DB_DATA_SOURCE: {valueFrom: {secretKeyRef: {name: pg-default-pguser-softserve, key: pgbouncer-uri}}} - SOFT_SERVE_INITIAL_ADMIN_KEYS: | - ${SECRET_ADMIN_SSH_PUBKEY_1} - ${SECRET_ADMIN_SSH_PUBKEY_2} - ${SECRET_ADMIN_SSH_PUBKEY_3} - ${SECRET_ADMIN_SSH_PUBKEY_4} + controllers: + main: + type: deployment + replicas: 1 + pod: + labels: + db.home.arpa/pg: "pg-default" + ingress.home.arpa/world: "allow" + ingress.home.arpa/cluster: "allow" + containers: + main: + image: &img + repository: "ghcr.io/charmbracelet/soft-serve" + tag: "v0.7.4@sha256:1e89075219cb95f62e912fb1831b170bcec73a30e2bb6431533e12dfca2666b0" + env: + TZ: "${CONFIG_TZ}" + SOFT_SERVE_NAME: "BioFlurry" + SOFT_SERVE_LOG_FORMAT: "json" + SOFT_SERVE_SSH_LISTEN_ADDR: ":23231" + SOFT_SERVE_SSH_PUBLIC_URL: "ssh://${APP_DNS_SOFT_SERVE}:22" + SOFT_SERVE_SSH_KEY_PATH: "/secrets/host-priv" + SOFT_SERVE_SSH_CLIENT_KEY_PATH: "/secrets/client-priv" + SOFT_SERVE_GIT_LISTEN_ADDR: ":9418" + SOFT_SERVE_HTTP_LISTEN_ADDR: ":23232" + SOFT_SERVE_HTTP_PUBLIC_URL: "https://${APP_DNS_SOFT_SERVE}" + SOFT_SERVE_HTTP_TLS_CERT_PATH: "/tls/tls.crt" + SOFT_SERVE_HTTP_TLS_KEY_PATH: "/tls/tls.key" + SOFT_SERVE_STATS_LISTEN_ADDR: ":23233" + SOFT_SERVE_LFS_ENABLED: "true" + SOFT_SERVE_LFS_SSH_ENABLED: "true" + SOFT_SERVE_DB_DRIVER: "postgres" + SOFT_SERVE_DB_DATA_SOURCE: {valueFrom: {secretKeyRef: {name: pg-default-pguser-softserve, key: pgbouncer-uri}}} + SOFT_SERVE_INITIAL_ADMIN_KEYS: {valueFrom: {secretKeyRef: {name: soft-serve-secrets, key: admin-keys}}} + securityContext: &sc + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: "10m" + memory: "128Mi" + limits: + cpu: "3000m" + memory: "1Gi" service: main: enabled: true primary: true - type: ClusterIP - ports: - http: - port: 23232 - stats: - port: 23233 - expose: - enabled: true + controller: main type: LoadBalancer externalTrafficPolicy: Cluster annotations: coredns.io/hostname: "${APP_DNS_SOFT_SERVE}" "io.cilium/lb-ipam-ips": "${APP_IP_SOFT_SERVE}" ports: + http: + enabled: true + port: 443 + targetPort: 23232 + protocol: HTTPS ssh: enabled: true port: 22 @@ -73,41 +86,50 @@ spec: port: 9418 targetPort: 9418 protocol: TCP - ingress: - main: - enabled: true - primary: true - ingressClassName: "nginx-internal" - hosts: - - host: &host "${APP_DNS_SOFT_SERVE_HTTPS}" - paths: - - path: / - pathType: Prefix - tls: - - hosts: - - *host - podSecurityContext: - runAsUser: &uid ${APP_UID_SOFT_SERVE} - runAsGroup: *uid - fsGroup: *uid - fsGroupChangePolicy: Always + stats: + port: 23233 persistence: - data: - enabled: true - type: nfs - server: "${IP_TRUENAS}" - path: "${PATH_NAS_PERSIST_K8S}/soft-serve" - mountPath: "/soft-serve" - ssh: + config: enabled: true type: secret - name: soft-serve-ssh - mountPath: /ssh - readOnly: true - resources: - requests: - cpu: 10m - memory: 128Mi - limits: - cpu: 3000m - memory: 6000Mi \ No newline at end of file + name: "soft-serve-secrets" + globalMounts: + - path: "/secrets" + data: + enabled: true + existingClaim: "soft-serve-data" + globalMounts: + - subPath: "data" + path: "/soft-serve" + tls: + enabled: true + type: secret + name: "short-domain-tls" + defaultMode: 0400 + globalMounts: + - path: "/tls" + readOnly: true + defaultPodOptions: + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + runAsNonRoot: true + runAsUser: &uid ${APP_UID_SOFT_SERVE} + runAsGroup: *uid + fsGroup: *uid + fsGroupChangePolicy: "Always" + seccompProfile: { type: "RuntimeDefault" } + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: "DoNotSchedule" + labelSelector: + matchLabels: + app.kubernetes.io/name: *app + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "fuckoff.home.arpa/soft-serve" + operator: "DoesNotExist" diff --git a/kube/deploy/apps/soft-serve/app/netpol.yaml b/kube/deploy/apps/soft-serve/app/netpol.yaml deleted file mode 100644 index dcc9f0f3..00000000 --- a/kube/deploy/apps/soft-serve/app/netpol.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: &app soft-serve - namespace: *app -spec: - endpointSelector: {} - ingress: - # same namespace - - fromEndpoints: - - matchLabels: - io.kubernetes.pod.namespace: *app - - fromEntities: - - "world" - toPorts: - - ports: - - port: "22" - protocol: TCP - - ports: - - port: "23231" - protocol: TCP - - ports: - - port: "9418" - protocol: TCP - egress: - # same namespace - - toEndpoints: - - matchLabels: - io.kubernetes.pod.namespace: *app diff --git a/kube/deploy/apps/soft-serve/app/ssh.yaml b/kube/deploy/apps/soft-serve/app/ssh.yaml deleted file mode 100644 index fb56e3f3..00000000 --- a/kube/deploy/apps/soft-serve/app/ssh.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: soft-serve-ssh - namespace: soft-serve -type: Opaque -stringData: - host-priv: ENC[AES256_GCM,data:mKJd9FH2Z9GyxSS0mTdiauImrobI+kqZ013ovtSNW46rEI59OAHuVwv/k/82FzTvCg/Aju4tqTfPhP6/Ivf/q0G1WRdAbRKOGdJ0picJ9Z3ETv6QQ1FVSkHCHmvW8BBG2BrHk80RcS3bx6VQa2v2D9KnFOEtdmUn5Afi6v8232B/ShvfX2QnSVuEH2WCiDJpZYp/vPRDC5deJqR1kYrlO0w/dSdo73LD/jU3XMxlOwLTaYnUIt+f8upyOY7e7x4EIKnE3E9DslG5LEdlG9hXDgGi+s6droYhAd70FGDrFTwguu3slMutt6Jr4nS4HfYsnBnkdhFmAqosWOYzMAarPxFbaPPVYJJK+acGn+iABCVXHE3bOx2Tr6MWAaSUvBi5n9SvdnI/CypzCfNiuDfWlrnv00OJLzgZSsmBSApUaA3oYkOYwJ6pelmajYMY3+wbIRWkxPFimU0Z+AcjVrOYkcub35UNPgzkYoB3uU4HmXjlVFWaIngURWQMe6AlPm6uPRZvh+2vz1xjBhC6rQN7pW2gMM0eq7xL8ZE3Riqbl/0s6sXba1XGWVfIKWXpYQsAJAc7xep5o+vburBB,iv:XahBh/d3u7DCc2jWvDufio9yqkxBRd0NOKJCmzYlrOI=,tag:4//267lH80qX/KakAHgZVw==,type:str] - client-priv: ENC[AES256_GCM,data:A/zmVmVXajbRkEsM7YHCAwF3bqBGGFhYH5eDF/KdnSyIy7Vd8Aa1txjpymLqNFbIwoHbvRZT4+bHVpD0Cd8jqSaQ6EnHv9jPQokLXxi/L5iyboCfILYGNyAa+guojK8sekJhGl5pSpQdhiwcff7JejbZNJXsuHVysZthnFobx72dh07zJ8rYO2ARBs530VLe5fcCH1RmmWj902B28RT3mMI8ZsesIzx41LucWM+z6n8AskifD15hrobImkLeIVNbaKnDxq04BDznCeu/t9wRKEw7hgMrzh0CREzWaIDv/GlSbpIQjhIdmUr9sgsMEs0gEom2pNy64yfpizEQP6JiOSmQ41wnqF3MR4AwM/B6mHkfbhJVt1B1O3zU2xXe7qN9I/k6SgopCBJtB621Ytzx4cZWZDwjdVfbUUWUChVxvvwV7wNTbGa7M9BaKENrVvvYHqC35Wlb+SlsxpQN+jjApYq85iS7DW0qSLk39JUCG+ZZ7ZTIw7X7x4ABLJ+ZHaQiZRMTH95rTqzqUHFWTMHWA1HeRkwSQklBGJHFO/fFOot5k0PVkuiSJRLvHo5//Pk6qn124uOPlhKNeuA=,iv:kUWc+NoR0GJ5AjqEJsjiEl1upWfwcbvFF+Itel9zdvQ=,tag:dudsRIBvkuVHbm0mqUCRnA==,type:str] -sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: - - recipient: age1u57l4s400gqstc0p485j4646cemntufr0pcyp32yudklsp90xpmszxvnkj - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwdkhXVUlzZkVoUkhSWjM2 - b055TWlUSTFlbUZaQzFyWnJnYVNKVlFzVEZ3Cnc2eHQ5N0ZHMGx1eXduSnljcmg3 - NmIwWjdyY1ljZVErNVJyTUZpTWJaa28KLS0tIDlkL3Q2VDZWeDBqMmphK3dzOWJI - dlJ2TWpPSTd6UExmWDdCZlNkNW5IYWcKXJpSqhh7Os21uI81t2t0LXOgrahJ85JK - +S3tdCFxmt8dsgNT/X0fjl5klGFp6KAd9VZI1D0MdF99kkZnwqGH/A== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2023-08-07T02:23:25Z" - mac: ENC[AES256_GCM,data:IgrrnGr0ixfYYlHRzSIRgBQneMHF+NFHB3rDYntcvvcIFYreNqAnGzEemhJ2efC8FucNifi1WOS4LFZmTN/yWTUoW2GOIcTL/1+AsnVw9NHvXqMOWr5Ltd4SJCHB7zYjhzsMtUW5+8EcqAgqod5S3MrBeTfQOxulqudPDIfsE0k=,iv:Oc9Tb+I1fKNdOOU07eBNKpRZiG4a76y7xydEdeA5aPs=,tag:hwnbxnv7JEHd0u2seYyk0A==,type:str] - pgp: - - created_at: "2023-08-07T02:19:08Z" - enc: | - -----BEGIN PGP MESSAGE----- - - hF4DAAAAAAAAAAASAQdACBYfXdyXxsT7K4N5TzgLabHY7oYm0lyps1HahU4XzEIw - QmMFwMUkpDPJNmM/uYG5zASXrnk5y53boDJdNObUdxH+8tx5fUlCDVfK6Rhx2I+H - 0l4B2H5UlRyDCONzp4N49SDUv2Gd33ZOpZDDy+B3x9GZ6SYSVZ8RNRLCSFYzP5C8 - 3P4lzFQzAkeLKCCmT7CPHgePcui91cOJ/EMc/gzGWYZBdIXXCyzXESZSW2y4wm/6 - =S81P - -----END PGP MESSAGE----- - fp: 31E70E5BC80C58AFF5DD649921AC5A1AC6E5B7F2 - encrypted_regex: ^(data|stringData)$ - version: 3.7.3 diff --git a/kube/deploy/apps/soft-serve/ks.yaml b/kube/deploy/apps/soft-serve/ks.yaml index 830f1093..19d1a768 100644 --- a/kube/deploy/apps/soft-serve/ks.yaml +++ b/kube/deploy/apps/soft-serve/ks.yaml @@ -4,23 +4,58 @@ kind: Kustomization metadata: name: soft-serve-app namespace: flux-system - labels: - prune.flux.home.arpa/enabled: "true" + labels: &l + app.kubernetes.io/name: "soft-serve" spec: - prune: true + commonMetadata: + labels: *l path: ./kube/deploy/apps/soft-serve/app + targetNamespace: "soft-serve" dependsOn: - name: soft-serve-db + - name: soft-serve-pvc +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: soft-serve-pvc + namespace: flux-system + labels: &l + app.kubernetes.io/name: "soft-serve" +spec: + commonMetadata: + labels: *l + path: ./kube/deploy/core/storage/volsync/template + targetNamespace: "soft-serve" + dependsOn: + - name: 1-core-storage-volsync-app + - name: 1-core-storage-rook-ceph-cluster + postBuild: + substitute: + PVC: "soft-serve-data" + SIZE: "20Gi" + SC: &sc "file" + SNAP: *sc + ACCESSMODE: "ReadWriteMany" + RUID: !!str &uid | + ${APP_UID_SOFT_SERVE} + RGID: !!str | + ${APP_UID_SOFT_SERVE} + RFSG: !!str | + ${APP_UID_SOFT_SERVE} --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: soft-serve-db namespace: flux-system - labels: + labels: &l prune.flux.home.arpa/enabled: "true" db.home.arpa/pg: "pg-default" + app.kubernetes.io/name: "soft-serve" spec: + commonMetadata: + labels: *l path: ./kube/deploy/core/db/pg/clusters/template/pguser targetNamespace: "pg" dependsOn: diff --git a/kube/deploy/apps/soft-serve/ns.yaml b/kube/deploy/apps/soft-serve/ns.yaml index f8c2c72e..0f624dc1 100644 --- a/kube/deploy/apps/soft-serve/ns.yaml +++ b/kube/deploy/apps/soft-serve/ns.yaml @@ -3,3 +3,8 @@ apiVersion: v1 kind: Namespace metadata: name: soft-serve + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + pod-security.kubernetes.io/enforce: &ps restricted + pod-security.kubernetes.io/audit: *ps + pod-security.kubernetes.io/warn: *ps diff --git a/kube/deploy/apps/soft-serve/volsync.yaml b/kube/deploy/apps/soft-serve/volsync.yaml deleted file mode 100644 index 77cda3e5..00000000 --- a/kube/deploy/apps/soft-serve/volsync.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: soft-serve-restic - namespace: soft-serve -type: Opaque -stringData: - RESTIC_REPOSITORY: ${SECRET_VOLSYNC_R2_REPO}/soft-serve - RESTIC_PASSWORD: ${SECRET_VOLSYNC_PASSWORD} - AWS_ACCESS_KEY_ID: ${SECRET_VOLSYNC_R2_ID} - AWS_SECRET_ACCESS_KEY: ${SECRET_VOLSYNC_R2_KEY} ---- -apiVersion: volsync.backube/v1alpha1 -kind: ReplicationSource -metadata: - name: soft-serve-restic - namespace: soft-serve -spec: - sourcePVC: config-soft-serve-0 # TODO: find PVC name - trigger: - schedule: "0 22 * * *" # 6am GMT+8 - restic: - copyMethod: Snapshot - pruneIntervalDays: 14 - repository: soft-serve-restic - cacheCapacity: 2Gi - volumeSnapshotClassName: block - storageClassName: block - moverSecurityContext: - runAsUser: &uid ${APP_UID_SOFT_SERVE} - runAsGroup: ${APP_UID_SOFT_SERVE} - fsGroup: ${APP_UID_SOFT_SERVE} - retain: - daily: 14 - within: 7d