feat(rook-ceph): add external users

Signed-off-by: JJGadgets <git@jjgadgets.tech>

kube/1-clusters/Biohazard/2-config/4-vars.yaml

@@ -29,6 +29,7 @@ data:
     DNS_OLD_DOCKER: ENC[AES256_GCM,data:uDLk+qfZlM9FkJ7uWP1ZYWD0wdIG,iv:iHJojVMWN6cq2XdvQLMsODrVeLhhn/Cqt5ZGr/ONy2A=,tag:3WuGLTQirXUjfiY1rIYcgA==,type:str]
     PATH_NAS_MEDIA: ENC[AES256_GCM,data:fzeT0pUx/geFxfnY67ZwUgAOF1r13bjSxFCCQz+1,iv:nYFnXgfJWl8ZPpxleet1Yq19t+6ncVkrmGyhGSchSxE=,tag:uj9grinnmKB1xKC2LwrPkA==,type:str]
     APP_IP_RADOSGW: ENC[AES256_GCM,data:79oO927eM1X8MA==,iv:cbhtTynWbMIKM8yRGywO9OJWxyWabZzW8VgJQxpSZd8=,tag:OU5+0QEKqjvAwATFdIBwgg==,type:str]
+    APP_DNS_RGW_S3: ENC[AES256_GCM,data:fBY7hU4Fo2HX,iv:crGFgE9fg+kVHMc2NrwIPNjtYKI6vw3iZd0GaVYymp0=,tag:8xKHmECqb7J+lRC1BKAZUg==,type:str]
     APP_DNS_INGRESS_WILDCARD: ENC[AES256_GCM,data:7OG0ww6rUzU=,iv:5ig0dQIfSVxbQS7nuqQygRcBKk8UmBFxX0unVT9bdzE=,tag:mCOMUNFEZs5IFvVrRNpFiQ==,type:str]
     APP_IP_NGINX: ENC[AES256_GCM,data:9Kg5zjk+1XfUHg==,iv:dbO0hMMho8J3t0mz6Eb5uMDB3QUCjG5pXPdeuQUFbNE=,tag:ICGE5EVo27W0rUB+Jekf2Q==,type:str]
     APP_IP_K8S_GATEWAY: ENC[AES256_GCM,data:oakciyUzwLlGJsc=,iv:leuHfW59gWSDaEpaOEMGbSpGFtbzAnoRp4spLxlTEq0=,tag:vltbWvNKa4QvEgXXo58d/A==,type:str]
@@ -57,6 +58,12 @@ data:
     CONFIG_SANDSTORM_INIT_MAP: ENC[AES256_GCM,data:uaM2kX5hlN2BoQ==,iv:U2jmxP35cy/eWT1JTdfr6Z3b4NAzIHG55Kb4emoAin0=,tag:rNCaa5zwKHesrto092oUcg==,type:str]
     CONFIG_SANDSTORM_INIT_SCENARIO: ENC[AES256_GCM,data:OJVCFbvqWXuYUPvdCiwRngUzfw==,iv:1NkA4VaF/xUdudDD2W5dHEDw55dkzwo2sof5krinJz0=,tag:rmD5eZpnHpOcSJXel3AQbg==,type:str]
     CONFIG_ZEROTIER_ENDPOINT: ENC[AES256_GCM,data:We/k3H6tvdmYoZ+i27Lll3bLRhXquz3fvztDI9T4tPjRc4uhG6fkpoa04hEAJffZc7yWNFUzUycPAp0=,iv:B6QCm/4bR68QEudl5o9kwJ6OtQvn1RrWeS6/W+Iaf/Q=,tag:S5xCE5e97gsBId7tpQA/mQ==,type:str]
+    USERS_1_ID: ENC[AES256_GCM,data:d+gVpZ2++zMJ,iv:EfKZSpKm9NsGTU4/lyVmueULkg/Dx2We3Wr2M1DkH6Y=,tag:Lq789OomRXAHKVO21Qj0Iw==,type:str]
+    USERS_1_NAME: ENC[AES256_GCM,data:HUBTvrZQh+cC,iv:9uE5OqV55E1mMPN1jV4RKgCwPh5FvQge1+oegL2TADY=,tag:DaDmQyg+w/yJ2RLFTcGyjA==,type:str]
+    USERS_2_ID: ENC[AES256_GCM,data:6Z/3XWU=,iv:7aoHN0pTeluYm/Rh2yjPKejFyKosIT5ntpXJZVbxY1s=,tag:neSexW5qCUSH0txXv37KTw==,type:str]
+    USERS_2_NAME: ENC[AES256_GCM,data:+i35bJLaW4w=,iv:zYvn5k22T91E88Yo8Z6uvuEHo24XBaARdOlKujvAWzs=,tag:SGGEHHVKmqXZwMhyRZIIhw==,type:str]
+    USERS_3_ID: ENC[AES256_GCM,data:UJWthgQ=,iv:yP1SbGOkCTss5/RjNRFOLI1kxIDWMneiMwrBXt8lECw=,tag:KW+AtwPnChJTgWCrj9mIPw==,type:str]
+    USERS_3_NAME: ENC[AES256_GCM,data:89cfPpVUwIniXQ==,iv:Hrh3k31gtzJ9ZwRng2K5ExmEehMomrRw0Zaq/P9k3oc=,tag:OZc/mFYyx+7tiRMIMYqFDQ==,type:str]
     CLUSTER_NAME: ENC[AES256_GCM,data:UTNoF7TkZ/Le,iv:mkA1AMzFXq0XEbprrqFCVWEyU37m/2y0P2SDzjDyTmw=,tag:bmh3LiqDrLEYuCzH1TnJzw==,type:str]
     CLUSTER_NAME_LOWER: ENC[AES256_GCM,data:dxucmLtxUMJg,iv:tco3xaQ03sBsr845xNrJvrqBa06DN+UwCZZrQ7GHkhA=,tag:Q0EtxM/GSYPGGPHCL7loSw==,type:str]
 sops:
@@ -74,8 +81,8 @@ sops:
             SnpvS3RUUlFMM1dUNGZQNkVqQ2VqNDAKywch6CgtS1AFLYxfML5dB7/5V6qZ0ob1
             63vBpqjOza3EqvfNKo+UMtK/fRK0Q5jlpuI+0/z9VrxzKEWsgUCBVQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-03-12T21:25:41Z"
-    mac: ENC[AES256_GCM,data:Ktg9ET9peepOEV0KXCg7cefVm87jE5tWIKtavJ6RUJRgAAMsynq2MV5VLSVwtFEmBKpH2oyCPSRGWeIgpfZZAysGl+QLHc8lWLf+GBjeoBXDhGUc0n8n83LczJkklPiI/1t3//qxO+zldGmAYYvQW86TM1o/I6qv7x4psbhPVNo=,iv:eOgIOoK1eQkWhD4GXxGH4Xe6QdeNCIRRVf+qIa+vYcY=,tag:KMxkHCuO/R/BUag4STXxuA==,type:str]
+    lastmodified: "2023-03-17T03:51:17Z"
+    mac: ENC[AES256_GCM,data:JJA83viVTyMTv+2E27fKBLBk3wn0jNrRSEBZvmsXVTwj5S3dZEM+OpfT3Eqm2XSJPcjDgjI5YHItFkksIr5b3+/qc7VsJ6szBL/dEcFQazzwTNhqr/KVicXDpU+vJ9JdwzLqmnTiUH9U2+gsOh4+uj4EjRqXrl3Fe9FmrFU+oTs=,iv:HqhyQSpvxFHZ/3DumhjDv1HxJ013L2CNW5lI3VPzN2I=,tag:IEphaFPVfcePIbasqdFy9Q==,type:str]
     pgp:
         - created_at: "2023-02-22T08:12:31Z"
           enc: |

kube/1-clusters/Biohazard/2-config/ceph-rgw-ext-users.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: jjgadgets
+  namespace: rook-ceph
+spec:
+  store: biohazard
+  displayName: "JJGadgets"

kube/1-clusters/Biohazard/2-config/kustomization.yaml

@@ -7,3 +7,4 @@ resources:
   - 3-secrets.yaml
   - 4-vars.yaml
   - 5-deploy.yaml
+  - ceph-rgw-ext-users.yaml

kube/3-deploy/1-core/02-storage/rook-ceph/app/helm-release.yaml

@@ -24,7 +24,7 @@ spec:
         namespace: flux-system
   values:
     enableCSIHostNetwork: true
-    useOperatorHostNetwork: true
+    useOperatorHostNetwork: false
     pspEnable: false
     crds:
       enabled: false

kube/3-deploy/1-core/02-storage/rook-ceph/cluster/object.yaml

@@ -45,8 +45,9 @@ spec:
     name: "${CLUSTER_NAME_LOWER}"
   preservePoolsOnDelete: true
   gateway:
-    sslCertificateRef: radosgw-tls
-    securePort: 443
+    # sslCertificateRef: radosgw-tls
+    port: 6953
+    # securePort: 443
     instances: 2
     service:
       annotations:
@@ -59,32 +60,30 @@ spec:
         memory: "1024Mi"
     priorityClassName: system-cluster-critical
 ---
-apiVersion: v1
-kind: Service
+apiVersion: networking.k8s.io/v1
+kind: Ingress
 metadata:
-  annotations:
-    io.cilium/lb-ipam-ips: 10.12.34.3
-  labels:
-    app: rook-ceph-rgw
-    ceph_daemon_id: biohazard
-    rgw: biohazard
-    rook_cluster: rook-ceph
-    rook_object_store: biohazard
-  name: rook-ceph-rgw-biohazard
+  name: &app rgw
   namespace: rook-ceph
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: 100m
+    nginx.ingress.kubernetes.io/server-alias: '*.${APP_DNS_RGW_S3}'
+  labels:
+    app.kubernetes.io/name: *app
+    app.kubernetes.io/instance: *app
 spec:
-  type: LoadBalancer
-  externalIPs:
-    - 10.12.34.3
-  ports:
-  - name: https
-    port: 443
-    protocol: TCP
-    targetPort: 443
-  selector:
-    app: rook-ceph-rgw
-    ceph_daemon_id: biohazard
-    rgw: biohazard
-    rook_cluster: rook-ceph
-    rook_object_store: biohazard
-  externalTrafficPolicy: Local
+  ingressClassName: nginx
+  rules:
+    - host: &host ${APP_DNS_RGW_S3}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: rook-ceph-rgw-biohazard
+                port:
+                  number: 6953
+  tls:
+    - hosts:
+        - *host