diff --git a/packages/extra/seaweedfs/Makefile b/packages/extra/seaweedfs/Makefile index a2504ed5..78e21e04 100644 --- a/packages/extra/seaweedfs/Makefile +++ b/packages/extra/seaweedfs/Makefile @@ -4,4 +4,4 @@ include ../../../scripts/package.mk generate: readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md - yq -o json -i '.properties.topology.enum = ["Simple","MultiZone"]' values.schema.json + yq -o json -i '.properties.topology.enum = ["Simple","MultiZone","Client"]' values.schema.json diff --git a/packages/extra/seaweedfs/README.md b/packages/extra/seaweedfs/README.md index f52bc9d3..0c6f9075 100644 --- a/packages/extra/seaweedfs/README.md +++ b/packages/extra/seaweedfs/README.md @@ -4,12 +4,14 @@ ### Common parameters -| Name | Description | Value | -| ------------------- | ------------------------------------------------------------------------------------------------------ | -------- | -| `host` | The hostname used to access the SeaweedFS externally (defaults to 's3' subdomain for the tenant host). | `""` | -| `topology` | The topology of the SeaweedFS cluster. (allowed values: Simple, MultiZone) | `Simple` | -| `replicationFactor` | The number of replicas for each volume in the SeaweedFS cluster. | `2` | -| `replicas` | Persistent Volume size for SeaweedFS | `2` | -| `size` | Persistent Volume size | `10Gi` | -| `storageClass` | StorageClass used to store the data | `""` | -| `zones` | A map of zones for MultiZone topology. Each zone can have its own number of replicas and size. | `{}` | +| Name | Description | Value | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------ | --------------- | +| `host` | The hostname used to access the SeaweedFS externally (defaults to 's3' subdomain for the tenant host). | `""` | +| `topology` | The topology of the SeaweedFS cluster. (allowed values: Simple, MultiZone, Client) | `Simple` | +| `replicationFactor` | The number of replicas for each volume in the SeaweedFS cluster. | `2` | +| `replicas` | Persistent Volume size for SeaweedFS | `2` | +| `size` | Persistent Volume size | `10Gi` | +| `storageClass` | StorageClass used to store the data | `""` | +| `zones` | A map of zones for MultiZone topology. Each zone can have its own number of replicas and size. | `{}` | +| `filer.external` | Enable external access to the SeaweedFS filer from outside the cluster. Use this when `topology` is not set to `Client`. | `false` | +| `remoteEndpoint` | The endpoint of the remote filer GRPC service. Used when `topology` is set to `Client`. | `1.2.3.4:18888` | diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag new file mode 100644 index 00000000..55b13e47 --- /dev/null +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -0,0 +1 @@ +ghcr.io/cozystack/cozystack/objectstorage-sidecar diff --git a/packages/extra/seaweedfs/templates/cm.yaml b/packages/extra/seaweedfs/templates/cm.yaml new file mode 100644 index 00000000..1aec9a1b --- /dev/null +++ b/packages/extra/seaweedfs/templates/cm.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: "{{ .Release.Name }}-deployed-topology" +data: + topology: {{ quote .Values.topology }} diff --git a/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml b/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml index 02c9b2aa..ec8860b7 100644 --- a/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml +++ b/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml @@ -3,6 +3,7 @@ kind: Role metadata: name: {{ .Release.Name }}-dashboard-resources rules: +{{- if not (eq .Values.topology "Client") }} - apiGroups: - "" resources: @@ -27,13 +28,15 @@ rules: - {{ $.Release.Name }}-volume - {{ $.Release.Name }}-db verbs: ["get", "list", "watch"] +{{- end }} + --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ .Release.Name }}-dashboard-resources subjects: -{{ include "cozy-lib.rbac.subjectsForTenantAndAccessLevel" (list "admin" .Release.Namespace) }} +{{ include "cozy-lib.rbac.subjectsForTenantAndAccessLevel" (list "super-admin" .Release.Namespace) }} roleRef: kind: Role name: {{ .Release.Name }}-dashboard-resources diff --git a/packages/extra/seaweedfs/templates/external/cosi-bucket-class.yaml b/packages/extra/seaweedfs/templates/external/cosi-bucket-class.yaml new file mode 100644 index 00000000..76eda0dc --- /dev/null +++ b/packages/extra/seaweedfs/templates/external/cosi-bucket-class.yaml @@ -0,0 +1,16 @@ +{{- if eq .Values.topology "Client" }} +--- +kind: BucketClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ .Release.Namespace }} +driverName: {{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io +deletionPolicy: Delete +--- +kind: BucketAccessClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ .Release.Namespace }} +driverName: {{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io +authenticationType: KEY +{{- end }} diff --git a/packages/extra/seaweedfs/templates/external/cosi-cluster-role.yaml b/packages/extra/seaweedfs/templates/external/cosi-cluster-role.yaml new file mode 100644 index 00000000..4ee30f30 --- /dev/null +++ b/packages/extra/seaweedfs/templates/external/cosi-cluster-role.yaml @@ -0,0 +1,61 @@ +{{- if eq .Values.topology "Client" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Namespace }}-objectstorage-provisioner +rules: +- apiGroups: ["objectstorage.k8s.io"] + resources: + - "buckets" + - "bucketaccesses" + - "bucketclaims" + - "bucketclasses" + - "bucketclasses/status" + - "bucketaccessclasses" + - "buckets/status" + - "bucketaccesses/status" + - "bucketclaims/status" + - "bucketaccessclasses/status" + verbs: + - "get" + - "list" + - "watch" + - "update" + - "create" + - "delete" +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: + - "get" + - "watch" + - "list" + - "delete" + - "update" + - "create" +- apiGroups: [""] + resources: + - "secrets" + - "events" + verbs: + - "get" + - "list" + - "watch" + - "update" + - "create" + - "delete" + - "patch" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Namespace }}-objectstorage-provisioner +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-objectstorage-provisioner + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Namespace }}-objectstorage-provisioner + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/packages/extra/seaweedfs/templates/external/cosi-deployment.yaml b/packages/extra/seaweedfs/templates/external/cosi-deployment.yaml new file mode 100644 index 00000000..720568a5 --- /dev/null +++ b/packages/extra/seaweedfs/templates/external/cosi-deployment.yaml @@ -0,0 +1,88 @@ +{{- if eq .Values.topology "Client" }} +{{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }} +{{- $ingress := index $myNS.metadata.annotations "namespace.cozystack.io/ingress" }} +{{- $host := index $myNS.metadata.annotations "namespace.cozystack.io/host" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $.Release.Name }}-objectstorage-provisioner + namespace: {{ $.Release.Namespace }} + labels: + app.kubernetes.io/component: objectstorage-provisioner + app.kubernetes.io/instance: seaweedfs + app.kubernetes.io/name: {{ $.Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: objectstorage-provisioner + app.kubernetes.io/instance: seaweedfs + app.kubernetes.io/name: {{ $.Release.Name }} + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + policy.cozystack.io/allow-to-apiserver: "true" + app.kubernetes.io/component: objectstorage-provisioner + app.kubernetes.io/instance: seaweedfs + app.kubernetes.io/name: {{ $.Release.Name }} + spec: + containers: + - name: seaweedfs-cosi-driver + image: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.1.2 + imagePullPolicy: IfNotPresent + env: + - name: DRIVERNAME + value: {{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io + - name: ENDPOINT + value: https://{{ .Values.host | default (printf "s3.%s" $host) }} + - name: SEAWEEDFS_FILER + value: {{ .Values.remoteEndpoint }} + - name: WEED_GRPC_CLIENT_KEY + value: /usr/local/share/ca-certificates/client/tls.key + - name: WEED_GRPC_CLIENT_CERT + value: /usr/local/share/ca-certificates/client/tls.crt + - name: WEED_GRPC_CA + value: /usr/local/share/ca-certificates/client/ca.crt + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /var/lib/cosi + name: socket + - mountPath: /usr/local/share/ca-certificates/client/ + name: client-cert + readOnly: true + - name: seaweedfs-cosi-sidecar + image: "{{ $.Files.Get "images/objectstorage-sidecar.tag" | trim }}" + imagePullPolicy: IfNotPresent + args: + - --v=5 + env: + - name: POD_NAMESPACE + value: {{ .Release.Namespace }} + volumeMounts: + - mountPath: /var/lib/cosi + name: socket + enableServiceLinks: false + restartPolicy: Always + terminationGracePeriodSeconds: 10 + serviceAccountName: {{ .Release.Name }}-objectstorage-provisioner + volumes: + - name: socket + emptyDir: {} + - name: client-cert + secret: + defaultMode: 420 + secretName: seaweedfs-client-cert +{{- end }} diff --git a/packages/extra/seaweedfs/templates/external/cosi-service-account.yaml b/packages/extra/seaweedfs/templates/external/cosi-service-account.yaml new file mode 100644 index 00000000..20407dee --- /dev/null +++ b/packages/extra/seaweedfs/templates/external/cosi-service-account.yaml @@ -0,0 +1,8 @@ +{{- if eq .Values.topology "Client" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-objectstorage-provisioner + namespace: {{ .Release.Namespace }} +automountServiceAccountToken: true +{{- end }} diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index eafdb4e3..34741f47 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -1,6 +1,9 @@ {{- /* Preflight checks for Helm template */ -}} -{{- if not (has .Values.topology (list "Simple" "MultiZone")) }} -{{- fail "Invalid value for .Values.topology. Must be one of 'Simple' or 'MultiZone'." }} +{{- if not (has .Values.topology (list "Simple" "MultiZone" "Client")) }} +{{- fail "Invalid value for .Values.topology. Must be one of 'Simple', 'MultiZone' or 'Client'." }} +{{- end }} +{{- if and (eq .Values.topology "Client") (not .Values.remoteEndpoint) }} +{{- fail "When topology is 'Client', .Values.remoteEndpoint must be set to a valid remote filer GRPC service endpoint." }} {{- end }} {{- if lt (int .Values.replicationFactor) 1 }} {{- fail "Invalid value for .Values.replicationFactor. Must be at least 1." }} @@ -13,16 +16,24 @@ {{- fail "replicationFactor must be less than or equal to the number of zones defined in .Values.zones." }} {{- end }} {{- end }} -{{- if lookup "v1" "PersistentVolumeClaim" "" (printf "%s-data1-seaweedfs-volume-0" .Release.Name) }} -{{- if eq .Values.topology "MultiZone" }} -{{- fail "Not allowed to switch between Simple and MultiZone topologies after the first deployment." }} -{{- end }} + +{{- $detectedTopology := "Unknown" }} +{{- $configMap := lookup "v1" "ConfigMap" .Release.Namespace (printf "%s-deployed-topology" .Release.Name) }} +{{- if $configMap }} +{{- $detectedTopology = dig "data" "topology" "Unknown" $configMap }} {{- else }} -{{- if and (eq .Values.topology "Simple") (.Release.IsUpgrade) }} -{{- fail "Not allowed to switch between Simple and MultiZone topologies after the first deployment." }} +{{- if lookup "v1" "PersistentVolumeClaim" .Release.Namespace (printf "data1-%s-volume-0" .Release.Name) }} +{{- $detectedTopology = "Simple" }} +{{- else if lookup "apps/v1" "StatefulSet" .Release.Namespace (printf "%s-master" .Release.Name) }} +{{- $detectedTopology = "MultiZone" }} {{- end }} {{- end }} +{{- if not (has $detectedTopology (list .Values.topology "Unknown")) }} +{{- fail (printf "Not allowed to switch between topologies after the first deployment: %s" $detectedTopology) }} +{{- end }} + +{{- if not (eq .Values.topology "Client") }} {{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }} {{- $ingress := index $myNS.metadata.annotations "namespace.cozystack.io/ingress" }} {{- $host := index $myNS.metadata.annotations "namespace.cozystack.io/host" }} @@ -198,3 +209,4 @@ spec: cnpg.io/cluster: seaweedfs-db cnpg.io/podRole: instance version: {{ $.Chart.Version }} +{{- end }} diff --git a/packages/extra/seaweedfs/templates/svc-external.yaml b/packages/extra/seaweedfs/templates/svc-external.yaml new file mode 100644 index 00000000..835e098e --- /dev/null +++ b/packages/extra/seaweedfs/templates/svc-external.yaml @@ -0,0 +1,18 @@ +{{- if and (not (eq .Values.topology "Client")) (.Values.filer.external) }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $.Release.Name }}-filer-external +spec: + type: LoadBalancer + externalTrafficPolicy: Local + ports: + - name: swfs-filer-grpc + port: 18888 + protocol: TCP + targetPort: 18888 + selector: + app.kubernetes.io/component: filer + app.kubernetes.io/name: {{ $.Release.Name }} +{{- end }} diff --git a/packages/extra/seaweedfs/templates/vpa.yaml b/packages/extra/seaweedfs/templates/vpa.yaml index a0c6d6a7..acdd8f47 100644 --- a/packages/extra/seaweedfs/templates/vpa.yaml +++ b/packages/extra/seaweedfs/templates/vpa.yaml @@ -1,3 +1,4 @@ +{{- if not (eq .Values.topology "Client") }} apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: @@ -64,3 +65,4 @@ spec: maxAllowed: cpu: "1" memory: 2048Mi +{{- end }} diff --git a/packages/extra/seaweedfs/values.schema.json b/packages/extra/seaweedfs/values.schema.json index 2f6e48bd..7221d3fd 100644 --- a/packages/extra/seaweedfs/values.schema.json +++ b/packages/extra/seaweedfs/values.schema.json @@ -1,10 +1,25 @@ { "properties": { + "filer": { + "properties": { + "external": { + "default": false, + "description": "Enable external access to the SeaweedFS filer from outside the cluster. Use this when `topology` is not set to `Client`.", + "type": "boolean" + } + }, + "type": "object" + }, "host": { "default": "", "description": "The hostname used to access the SeaweedFS externally (defaults to 's3' subdomain for the tenant host).", "type": "string" }, + "remoteEndpoint": { + "default": "1.2.3.4:18888", + "description": "The endpoint of the remote filer GRPC service. Used when `topology` is set to `Client`.", + "type": "string" + }, "replicas": { "default": 2, "description": "Persistent Volume size for SeaweedFS", @@ -27,11 +42,12 @@ }, "topology": { "default": "Simple", - "description": "The topology of the SeaweedFS cluster. (allowed values: Simple, MultiZone)", + "description": "The topology of the SeaweedFS cluster. (allowed values: Simple, MultiZone, Client)", "type": "string", "enum": [ "Simple", - "MultiZone" + "MultiZone", + "Client" ] }, "zones": { diff --git a/packages/extra/seaweedfs/values.yaml b/packages/extra/seaweedfs/values.yaml index 50846fcc..f7e379de 100644 --- a/packages/extra/seaweedfs/values.yaml +++ b/packages/extra/seaweedfs/values.yaml @@ -3,7 +3,7 @@ ## @param host The hostname used to access the SeaweedFS externally (defaults to 's3' subdomain for the tenant host). host: "" -## @param topology The topology of the SeaweedFS cluster. (allowed values: Simple, MultiZone) +## @param topology The topology of the SeaweedFS cluster. (allowed values: Simple, MultiZone, Client) ## topology: Simple @@ -31,3 +31,10 @@ storageClass: "" ## replicas: 2 ## size: 10Gi zones: {} + +## @param filer.external Enable external access to the SeaweedFS filer from outside the cluster. Use this when `topology` is not set to `Client`. +filer: + external: false + +## @param remoteEndpoint The endpoint of the remote filer GRPC service. Used when `topology` is set to `Client`. +remoteEndpoint: "1.2.3.4:18888" diff --git a/packages/system/objectstorage-controller/Makefile b/packages/system/objectstorage-controller/Makefile index b947ce36..d0b16bbf 100644 --- a/packages/system/objectstorage-controller/Makefile +++ b/packages/system/objectstorage-controller/Makefile @@ -27,3 +27,4 @@ image-controller image-sidecar: IMAGE="$(REGISTRY)/objectstorage-$(TARGET):$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/$(TARGET).json -r)" && \ yq -i '$(YAML_PATH) = strenv(IMAGE)' $(VALUES_FILE) rm -f images/$(TARGET).json + yq .seaweedfs.cosi.sidecar.image ../seaweedfs/values.yaml > ../../extra/seaweedfs/images/objectstorage-sidecar.tag