feat: add zipline

2026-03-20 21:39:40 +00:00 · 2023-06-03 14:39:05 +08:00
parent 951521ec33
commit 340b5203ab
9 changed files with 220 additions and 8 deletions
--- a/kube/1-clusters/Biohazard/2-config/kustomization.yaml
+++ b/kube/1-clusters/Biohazard/2-config/kustomization.yaml
@@ -25,6 +25,8 @@ resources:
  - ../../../3-deploy/2-apps/ntfy/
  - ../../../3-deploy/2-apps/satisfactory/
  - ../../../3-deploy/2-apps/headscale/
+  - ../../../3-deploy/2-apps/zipline/
+
 patches:
  - patch: |-
      apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
--- a/kube/3-deploy/1-core/01-networking/cilium/netpols/labelled-allow-egress.yaml
+++ b/kube/3-deploy/1-core/01-networking/cilium/netpols/labelled-allow-egress.yaml
@@ -19,7 +19,7 @@ spec:
    matchLabels:
      egress.home.arpa/apiserver: allow
  egress:
-    - toEnities:
+    - toEntities:
        - kube-apiserver
 ---
 apiVersion: cilium.io/v2
--- a/kube/3-deploy/2-apps/zipline/app/hr.yaml
+++ b/kube/3-deploy/2-apps/zipline/app/hr.yaml
@@ -0,0 +1,145 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &app zipline
+  namespace: *app
+spec:
+  chart:
+    spec:
+      chart: app-template
+      version: 1.5.1
+      sourceRef:
+        name: bjw-s
+        kind: HelmRepository
+        namespace: flux-system
+  values:
+    global:
+      fullnameOverride: *app
+    controller:
+      type: deployment
+      replicas: 1
+    image:
+      repository: ghcr.io/diced/zipline
+      tag: 3.7.0@sha256:6ff84bd03bde7d1f108ed87b3f953027d5dace33e2c8238ca1fbe432a499f723
+    podLabels:
+      ingress.home.arpa/nginx: "allow"
+      db.home.arpa/pg: "pg-default"
+      s3.home.arpa/store: "rgw-${CLUSTER_NAME_LOWER}"
+    env:
+      TZ: "${CONFIG_TZ}"
+      CORE_SECRET:
+        valueFrom:
+          secretKeyRef:
+            name: zipline-secrets
+            key: CORE_SECRET
+      CORE_DATABASE_URL:
+        valueFrom:
+          secretKeyRef:
+            name: zipline-secrets
+            key: CORE_DATABASE_URL
+      # TODO: 2023-06-03 add OAuth when Authentik support is merged: https://github.com/diced/zipline/discussions/321 and https://github.com/diced/zipline/pull/372/
+      DATASOURCE_S3_ACCESS_KEY_ID:
+        valueFrom:
+          secretKeyRef:
+            name: zipline-data-s3
+            key: AWS_ACCESS_KEY_ID
+      DATASOURCE_S3_SECRET_ACCESS_KEY:
+        valueFrom:
+          secretKeyRef:
+            name: zipline-data-s3
+            key: AWS_SECRET_ACCESS_KEY
+      DATASOURCE_TYPE: "s3"
+      DATASOURCE_S3_BUCKET: "zipline-data"
+      DATASOURCE_S3_ENDPOINT: "rook-ceph-rgw-${CLUSTER_NAME_LOWER}.rook-ceph.svc.cluster.local"
+      DATASOURCE_S3_PORT: "6953"
+      DATASOURCE_S3_REGION: "us-west-1"
+      DATASOURCE_S3_FORCE_S3_PATH: "true" # TODO: 2023-06-03 current in-cluster RGW doesn't use subdomain (VirtualHost) based S3 buckets yet, it uses path-based
+      DATASOURCE_S3_USE_SSL: "false"
+      CORE_RETURN_HTTPS: "true"
+      CORE_HTTPS: "true"
+      CORE_HOST: "0.0.0.0"
+      CORE_PORT: "3000"
+      CORE_COMPRESSION_ENABLED: "true"
+      CORE_COMPRESSION_THRESHOLD: "200mb"
+      CORE_COMPRESSION_ON_DASHBOARD: "true"
+      EXIF_ENABLED: "false"
+      EXIF_REMOVE_GPS: "true"
+      FEATURES_INVITES: "false"
+      FEATURES_INVITES_LENGTH: "16"
+      FEATURES_OAUTH_REGISTRATION: "true"
+      FEATURES_USER_REGISTRATION: "false"
+      FEATURES_HEADLESS: "false"
+      RATELIMIT_USER: "5"
+      RATELIMIT_ADMIN: "1"
+      UPLOADER_DEFAULT_FORMAT: "DATE"
+      UPLOADER_ROUTE: &upload "/u"
+      UPLOADER_ADMIN_LIMIT: "100gb"
+      UPLOADER_USER_LIMIT: "500mb"
+      UPLOADER_DISABLED_EXTENSIONS: "ps1,pdf,bat,exe,sh,fish"
+      UPLOADER_FORMAT_DATE: "YYYY-MM-DD_HH-mm-ss"
+      UPLOADER_DEFAULT_EXPIRATION: ""
+      URLS_ROUTE: &shorten "/l"
+      URLS_LENGTH: "6"
+      WEBSITE_TITLE: "JJGadgets Upload"
+      WEBSITE_SHOW_FILES_PER_USER: "false"
+      WEBSITE_EXTERNAL_LINKS: |
+        '[{"label":"Admin: JJGadgets","link":"https://jjgadgets.tech"},{"label":"Powered by Zipline","link":"https://github.com/diced/zipline"}]'
+      WEBSITE_SHOW_VERSION: "false"
+      WEBSITE_DISABLE_MEDIA_PREVIEW: "true"
+    service:
+      main:
+        ports:
+          http:
+            port: 3000
+    ingress:
+      main:
+        enabled: true
+        primary: true
+        ingressClassName: nginx
+        annotations:
+          external-dns.alpha.kubernetes.io/target: "${IP_EC2_INGRESS}"
+          nginx.ingress.kubernetes.io/proxy-body-size: "100m"
+        hosts:
+          - host: &host "${APP_DNS_ZIPLINE}"
+            paths:
+              - path: *upload
+                pathType: Prefix
+              - path: *shorten
+                pathType: Prefix
+        tls:
+          - hosts:
+              - *host
+      internal:
+        enabled: true
+        primary: false
+        ingressClassName: nginx
+        annotations:
+          nginx.ingress.kubernetes.io/proxy-body-size: "100m"
+          nginx.ingress.kubernetes.io/whitelist-source-range: "10.0.0.0/8, 100.64.0.0/10"
+        hosts:
+          - host: &host "${APP_DNS_ZIPLINE}"
+            paths:
+              - path: /
+                pathType: Prefix
+        tls:
+          - hosts:
+              - *host
+    podSecurityContext:
+      runAsUser: &uid ${APP_UID_ZIPLINE}
+      runAsGroup: *uid
+      fsGroup: *uid
+      fsGroupChangePolicy: Always
+    resources:
+      requests:
+        cpu: 10m
+        memory: 128Mi
+      limits:
+        memory: 6000Mi
+    initContainers:
+      01-init-db:
+        image: ghcr.io/onedr0p/postgres-init:14.8@sha256:d8391076d2c6449927a6409c4e72aaa5607c95be51969036f4feeb7c999638ea
+        imagePullPolicy: IfNotPresent
+        envFrom:
+          - secretRef:
+              name: zipline-pg-superuser
--- a/kube/3-deploy/2-apps/zipline/app/s3.yaml
+++ b/kube/3-deploy/2-apps/zipline/app/s3.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: zipline-data-s3
+  namespace: zipline
+spec:
+  bucketName: "zipline-data"
+  storageClassName: "rgw-${CLUSTER_NAME_LOWER}"
--- a/kube/3-deploy/2-apps/zipline/app/secret.yaml
+++ b/kube/3-deploy/2-apps/zipline/app/secret.yaml
@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "zipline-secrets"
+  namespace: "zipline"
+type: Opaque
+stringData:
+  CORE_SECRET: "${SECRET_ZIPLINE_CORE_SECRET}"
+  CORE_DATABASE_URL: "postgres://${SECRET_ZIPLINE_PG_USER}:${SECRET_ZIPLINE_PG_PASS}@pg-default-rw.pg.svc.cluster.local:5432/${SECRET_ZIPLINE_PG_DBNAME}"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "zipline-pg-superuser"
+  namespace: "zipline"
+type: Opaque
+stringData:
+  INIT_POSTGRES_HOST: "pg-default-rw.pg.svc.cluster.local"
+  INIT_POSTGRES_DBNAME: "${SECRET_ZIPLINE_PG_DBNAME}"
+  INIT_POSTGRES_USER: "${SECRET_ZIPLINE_PG_USER}"
+  INIT_POSTGRES_PASS: "${SECRET_ZIPLINE_PG_PASS}"
+  INIT_POSTGRES_SUPER_PASS: "${SECRET_PG_DEFAULT_SUPER_PASS}"
--- a/kube/3-deploy/2-apps/zipline/ks.yaml
+++ b/kube/3-deploy/2-apps/zipline/ks.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: zipline-app
+  namespace: flux-system
+spec:
+  path: ./kube/3-deploy/2-apps/zipline/app
+  dependsOn:
+    - name: ${CLUSTER_NAME_LOWER}-1-core-02-storage-rook-ceph
+    - name: ${CLUSTER_NAME_LOWER}-1-core-04-dns-internal
+    - name: ${CLUSTER_NAME_LOWER}-1-core-05-ingress-nginx
+    - name:  1-core-db-pg-clusters-default
+  healthChecks:
+    - name: zipline
+      namespace: zipline
+      kind: HelmRelease
+      apiVersion: helm.toolkit.fluxcd.io/v2beta1
--- a/kube/3-deploy/2-apps/zipline/kustomization.yaml
+++ b/kube/3-deploy/2-apps/zipline/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ns.yaml
+  - ks.yaml
--- a/kube/3-deploy/2-apps/zipline/ns.yaml
+++ b/kube/3-deploy/2-apps/zipline/ns.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: zipline
--- a/kube/clusters/biohazard/config/secrets.sops.env
+++ b/kube/clusters/biohazard/config/secrets.sops.env
@@ -47,12 +47,16 @@ SECRET_HEADSCALE_PG_PASS=ENC[AES256_GCM,data:9k7/jiIq29R7Qzm36IvaROJPQaTJM26JJsQ
 SECRET_HEADSCALE_OIDC_URL=ENC[AES256_GCM,data:Y0SHVAfaaCrRjHGdU5HAp+y6z3J0qWU20D4TsIH6xlOX7SC2tQiF3YukbhB0cJJk2LpkOQeV4DKn,iv:Uh/Zxij7aqe4/tsy+7h6K5HcgzYCh31pqKWWR/Bx8g8=,tag:YkQYT4yc03xPzWCRq1uzpQ==,type:str]
 SECRET_HEADSCALE_OIDC_ID=ENC[AES256_GCM,data:AeL8W43fbHiqElAAwCynaoJD8BwnSVH1EWlBO29ysf+vGhJ7CidDJQ==,iv:BrH92xlpI/ApKq8hg/0EylI6Fo76B5Eg6lWrWDfgei8=,tag:UXYlOY4T0Bh/q+ySRfW2ew==,type:str]
 SECRET_HEADSCALE_OIDC_SECRET=ENC[AES256_GCM,data:dJjmjRWQ12JCtXsGDJICIRMqdsc5HfpH+6fh6WgpOZ8tyNdrSRP2ow1Jag7zEMIeMyi+vh2T/s9DnnRFBbFQDcmon3WuI5xIjEmDU4/IPIkhoE4FKBjgHib6ML4IxcIFnq0haur3FJS1/StvcVONyrkaHOjAEs7Tazl61BlFQN0=,iv:eo0hsCL9K0HhMRFWeaoUs56f5zexnKoECnWL16o9YJM=,tag:tYMAZGLNqZ8dXJrlgpJRdw==,type:str]
-sops_lastmodified=2023-06-01T18:01:08Z
-sops_pgp__list_0__map_enc=-----BEGIN PGP MESSAGE-----\n\nhF4DAAAAAAAAAAASAQdANDTQwVjZ/Ad3iqBe0LL2sGCrEvrl6W6VaMjFgJCUkzYw\nwASmi9Y/OqREXtEItA1rKZDTM38LuMfcU4vAeEV0SNWlW5CQquN8UpLwMATrBdXr\n0lwBcvIZFLbbnfqFAdJ1EzbRWvHuh+yn5DBMH+odm3ZLaJqiiV9EaWhfl2rdIOr4\nPJQf6Ev1hueWmc9H45a8nvwH8sOl9MH9hl3TW7o9JOOhGmZ4BBVaSJW6f0UiZw==\n=iSQg\n-----END PGP MESSAGE-----\n
-sops_age__list_0__map_recipient=age1u57l4s400gqstc0p485j4646cemntufr0pcyp32yudklsp90xpmszxvnkj
-sops_mac=ENC[AES256_GCM,data:5rTK2F1MPlQllN7nUqUemmQamGCGWgu+o8pGgKA9ORGqt/iQ2W4Tt6qJ2nXMXFqkmAaafIhjBDcK1S1+ETuf3bNJrGVZFJCyVanmUpUh46DfvE8/ZlI478KWkvmN1Vpxa3wYfztbwHlrRBOm75R9duotXEQPayc6WOWEJzbgl5s=,iv:YhqRncD914TbU+qt/dj/5Pb3oaVhPKk9Dmdt1NlD7lQ=,tag:8aeKZn1pZX81klw3P3mtSQ==,type:str]
-sops_pgp__list_0__map_fp=31E70E5BC80C58AFF5DD649921AC5A1AC6E5B7F2
-sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxalh5ckhGWGxFTmFqSDQv\ndXlPOUlyYVNkWHA5VGN2TERvaWtWMHlJdFRNCnQ1NlJldEgxb2E0VEdVSDVpbHp5\nZEpTMEQ5dWU0Q2ZWTFBOZFp5Ti95ejQKLS0tIDF0c3VlazRzVWtVQ1JXT3hyTWNN\nWXpUSUNydGY4V04xZ2dTSzlvWmNOTGsKQ3rimeB7zqB4dYMp1pR1AOltXk+GhGsb\ns0jDxr/SiPUaiYoVCY4fqu9geXNRDGlPh3T2Lhs9Siif4Vnc8qTQBw==\n-----END AGE ENCRYPTED FILE-----\n
+SECRET_ZIPLINE_CORE_SECRET=ENC[AES256_GCM,data:7SaUQTrItXAPYmDL7n2PS16wuCCtOMqkHEG1u4J2jt86lWAvRi5egx71a15iQGtWkaXialPRlD6ok8Nw8CJsYTNeSMp5DPgMWX3455maFwK1scRf5EhFwO3NimEBGki/nfNrW9WPNePW9IW4jPgAfYlQ9a4wMVVN/xfA3760rL/fir4zrV9dQiv1JYaDXd4HcoopSBBvB/9wMjqJPmCOQXru+4A342cYcEpuzg==,iv:tE2qKJ2q1ejQ2f4kyMdwlUTdLyWOIKz6OjukyZDFmu4=,tag:BO1zBDh4c1Z9bQMDAFcTag==,type:str]
+SECRET_ZIPLINE_PG_DBNAME=ENC[AES256_GCM,data:1n/7RoIBs7aYFvj5j/gY1MY3cNb89/3HSNn8/rvVl1k8v03hBpt2nw==,iv:8hWz49X3Uh0E0NLNNdPEBJvUmIuBnfMS7urcYcjnHk0=,tag:pTMxq1umMkdAdHmrMyMTVQ==,type:str]
+SECRET_ZIPLINE_PG_USER=ENC[AES256_GCM,data:0Gywyf7j50UpHf1m/tmxYnJFYHptB1WbTSI8JXmJgvuqpGjtW4X7mm1tuB13XmF1JaeIDBWwuQdy+6YcYC7IwA==,iv:owyh+uvQGR2OvsfuV7agXaVY3nnvoNyrMyYP68Elb6E=,tag:mFsupknAyRTYnk3eL/2Baw==,type:str]
+SECRET_ZIPLINE_PG_PASS=ENC[AES256_GCM,data:mUvEAvWgzRR7580m0b6YYamZTMQbRbhdb0SSITbF7YrrejUXImqmUqhPiarVKwoFXCJNsxKXncJyPiYauQqtrqM/92MNqnqWYJlrswbOSIO5r+PnQYTaROGNPbQ5+aD+S6aMhOdoX/CdR/VQBr2Fj5+NC3C7hZCZDafIDnwmqQE=,iv:9Jd8vAemgxhQizAx+Cx6nFpBQIyErnJYrxLqmBhJ4zM=,tag:OsPY2dqQ9YBUvVGvZVIOTA==,type:str]
+sops_mac=ENC[AES256_GCM,data:K4WA+k8xuRa6YadE9tZuO49WFrmcEJgpF0vT0EaraA2Kf0iVRo5toiOLNclmY4+b27lOa7a0PDW+MucgCVoBpnD3YEVte0O7CgnfkZceyD79GQKzDomzSgzKVqiyaM3fJQZ+IzWe2kPwKwiE+X0vZ7wNAsgWCdFHIDhQJoqaSMk=,iv:1Vl4FmQCgAnFPhLSB6YYW5ztf8o8rHDXnAlQkMNSOlQ=,tag:dJkOG4pO/ipKz/asgbALlg==,type:str]
 sops_unencrypted_suffix=_unencrypted
-sops_version=3.7.3
+sops_lastmodified=2023-06-03T12:38:10Z
+sops_pgp__list_0__map_fp=31E70E5BC80C58AFF5DD649921AC5A1AC6E5B7F2
+sops_age__list_0__map_recipient=age1u57l4s400gqstc0p485j4646cemntufr0pcyp32yudklsp90xpmszxvnkj
+sops_pgp__list_0__map_enc=-----BEGIN PGP MESSAGE-----\n\nhF4DAAAAAAAAAAASAQdANDTQwVjZ/Ad3iqBe0LL2sGCrEvrl6W6VaMjFgJCUkzYw\nwASmi9Y/OqREXtEItA1rKZDTM38LuMfcU4vAeEV0SNWlW5CQquN8UpLwMATrBdXr\n0lwBcvIZFLbbnfqFAdJ1EzbRWvHuh+yn5DBMH+odm3ZLaJqiiV9EaWhfl2rdIOr4\nPJQf6Ev1hueWmc9H45a8nvwH8sOl9MH9hl3TW7o9JOOhGmZ4BBVaSJW6f0UiZw==\n=iSQg\n-----END PGP MESSAGE-----\n
 sops_pgp__list_0__map_created_at=2023-06-01T18:01:07Z
+sops_version=3.7.3
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxalh5ckhGWGxFTmFqSDQv\ndXlPOUlyYVNkWHA5VGN2TERvaWtWMHlJdFRNCnQ1NlJldEgxb2E0VEdVSDVpbHp5\nZEpTMEQ5dWU0Q2ZWTFBOZFp5Ti95ejQKLS0tIDF0c3VlazRzVWtVQ1JXT3hyTWNN\nWXpUSUNydGY4V04xZ2dTSzlvWmNOTGsKQ3rimeB7zqB4dYMp1pR1AOltXk+GhGsb\ns0jDxr/SiPUaiYoVCY4fqu9geXNRDGlPh3T2Lhs9Siif4Vnc8qTQBw==\n-----END AGE ENCRYPTED FILE-----\n