From edc12e3f7e816d0483787761f1cb5f3b5e325a24 Mon Sep 17 00:00:00 2001 From: Isaiah Olson Date: Mon, 15 Sep 2025 02:27:56 -0500 Subject: [PATCH] Add FoundationDB configuration values for storage engine and redundancy mode, update tests, and fix workload monitor Signed-off-by: Isaiah Olson --- hack/e2e-apps/foundationdb.bats | 23 ++++++++++++++----- packages/apps/foundationdb/README.md | 9 ++++++++ .../foundationdb/templates/_resources.tpl | 16 ++++++++++++- .../apps/foundationdb/templates/cluster.yaml | 23 +++++++++++-------- .../templates/workloadmonitor.yaml | 8 +++---- packages/apps/foundationdb/values.schema.json | 14 +++++++++++ packages/apps/foundationdb/values.yaml | 4 ++++ 7 files changed, 76 insertions(+), 21 deletions(-) diff --git a/hack/e2e-apps/foundationdb.bats b/hack/e2e-apps/foundationdb.bats index 99937a42..4c8d1b53 100644 --- a/hack/e2e-apps/foundationdb.bats +++ b/hack/e2e-apps/foundationdb.bats @@ -9,13 +9,14 @@ metadata: name: $name namespace: tenant-test spec: - replicas: 3 cluster: version: "7.3.63" processCounts: storage: 3 stateless: -1 cluster_controller: 1 + redundancyMode: "double" + storageEngine: "ssd-2" faultDomain: key: "foundationdb.org/none" valueFrom: "\$FDB_ZONE_ID" @@ -26,18 +27,18 @@ spec: backup: enabled: false s3: - bucket: "s3.example.org/fdb-backups" + bucket: "" endpoint: "" - region: "us-east-1" + region: "" credentials: - accessKeyId: "oobaiRus9pah8PhohL1ThaeTa4UVa7gu" - secretAccessKey: "ju3eum4dekeich9ahM1te8waeGai0oog" + accessKeyId: "" + secretAccessKey: "" retentionPolicy: "7d" monitoring: enabled: true customParameters: - "knob_disable_posix_kernel_aio=1" - imageType: "split" + imageType: "unified" automaticReplacements: true EOF sleep 15 @@ -102,6 +103,16 @@ EOF kubectl -n tenant-test get pod "$storage_pod" -o jsonpath='{.spec.containers[0].securityContext.runAsUser}' | grep -q '4059' kubectl -n tenant-test get pod "$storage_pod" -o jsonpath='{.spec.containers[0].securityContext.runAsGroup}' | grep -q '4059' + # Verify volumeClaimTemplate is properly configured in FoundationDBCluster CRD + timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.spec.processes.general.volumeClaimTemplate.spec.resources.requests.storage}' | grep -q '1Gi'; do sleep 10; done" + + # Verify PVCs are created with correct storage size (1Gi as specified in test) + timeout 120 sh -ec "until [ \$(kubectl -n tenant-test get pvc -l foundationdb.org/fdb-cluster-name=foundationdb-$name --no-headers | wc -l) -ge 3 ]; do sleep 10; done" + kubectl -n tenant-test get pvc -l foundationdb.org/fdb-cluster-name=foundationdb-$name -o jsonpath='{.items[*].spec.resources.requests.storage}' | grep -q '1Gi' + + # Verify actual PVC storage capacity matches requested size + kubectl -n tenant-test get pvc -l foundationdb.org/fdb-cluster-name=foundationdb-$name -o jsonpath='{.items[*].status.capacity.storage}' | grep -q '1Gi' + # Clean up kubectl -n tenant-test delete foundationdb $name diff --git a/packages/apps/foundationdb/README.md b/packages/apps/foundationdb/README.md index 6b0cb2bc..fed4acde 100644 --- a/packages/apps/foundationdb/README.md +++ b/packages/apps/foundationdb/README.md @@ -124,6 +124,13 @@ FoundationDB is designed for high availability: - Configurable fault domains for rack/zone awareness - Transaction log redundancy +The included `WorkloadMonitor` is automatically configured based on the `cluster.redundancyMode` value. It sets the `minReplicas` property on the `WorkloadMonitor` resource to ensure the cluster's health status accurately reflects its fault tolerance level. The number of tolerated failures is as follows: +- `single`: 0 failures +- `double`: 1 failure +- `triple` and datacenter-aware modes: 2 failures + +For example, with the default configuration (`redundancyMode: double` and 3 storage pods), `minReplicas` will be set to 2. + ## Performance Considerations - Use SSD storage for better performance @@ -149,6 +156,8 @@ For Cozystack-specific issues, consult the Cozystack documentation or support ch | `cluster.processCounts.storage` | Number of storage processes (determines cluster size) | `int` | `3` | | `cluster.processCounts.cluster_controller` | Number of cluster controller processes | `int` | `1` | | `cluster.version` | Version of FoundationDB to use | `string` | `7.3.63` | +| `cluster.redundancyMode` | Database redundancy mode (single, double, triple, three_datacenter, three_datacenter_fallback) | `string` | `double` | +| `cluster.storageEngine` | Storage engine (ssd-2, ssd-redwood-v1, ssd-rocksdb-v1, memory) | `string` | `ssd-2` | | `cluster.faultDomain` | Fault domain configuration | `object` | `{}` | | `cluster.faultDomain.key` | Fault domain key | `string` | `kubernetes.io/hostname` | | `cluster.faultDomain.valueFrom` | Fault domain value source | `string` | `spec.nodeName` | diff --git a/packages/apps/foundationdb/templates/_resources.tpl b/packages/apps/foundationdb/templates/_resources.tpl index e21e34e0..c1b4915e 100644 --- a/packages/apps/foundationdb/templates/_resources.tpl +++ b/packages/apps/foundationdb/templates/_resources.tpl @@ -30,4 +30,18 @@ Chart name and version */}} {{- define "foundationdb.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} \ No newline at end of file +{{- end }} + +{{/* +Calculate minReplicas for WorkloadMonitor based on redundancyMode +*/}} +{{- define "foundationdb.minReplicas" -}} +{{- $replicas := .Values.cluster.processCounts.storage -}} +{{- if or (eq .Values.cluster.redundancyMode "triple") (eq .Values.cluster.redundancyMode "three_data_hall") (eq .Values.cluster.redundancyMode "three_datacenter") (eq .Values.cluster.redundancyMode "three_datacenter_fallback") (eq .Values.cluster.redundancyMode "three_data_hall_fallback") }} +{{- print (max 1 (sub $replicas 2)) -}} +{{- else if eq .Values.cluster.redundancyMode "double" }} +{{- print (max 1 (sub $replicas 1)) -}} +{{- else }} +{{- print $replicas -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/packages/apps/foundationdb/templates/cluster.yaml b/packages/apps/foundationdb/templates/cluster.yaml index 209a78f9..06992cb5 100644 --- a/packages/apps/foundationdb/templates/cluster.yaml +++ b/packages/apps/foundationdb/templates/cluster.yaml @@ -11,7 +11,11 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} spec: version: {{ .Values.cluster.version | quote }} - + + databaseConfiguration: + redundancy_mode: {{ .Values.cluster.redundancyMode }} + storage_engine: {{ .Values.cluster.storageEngine }} + processCounts: {{- toYaml .Values.cluster.processCounts | nindent 4 }} @@ -76,15 +80,14 @@ spec: memory: 128Mi securityContext: {{- toYaml .Values.securityContext | nindent 16 }} - - volumeClaimTemplate: - spec: - {{- if .Values.storage.storageClass }} - storageClassName: {{ .Values.storage.storageClass }} - {{- end }} - resources: - requests: - storage: {{ .Values.storage.size }} + volumeClaimTemplate: + spec: + {{- if .Values.storage.storageClass }} + storageClassName: {{ .Values.storage.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.storage.size }} routing: dnsDomain: {{ $clusterDomain }} diff --git a/packages/apps/foundationdb/templates/workloadmonitor.yaml b/packages/apps/foundationdb/templates/workloadmonitor.yaml index b06fe877..306797f3 100644 --- a/packages/apps/foundationdb/templates/workloadmonitor.yaml +++ b/packages/apps/foundationdb/templates/workloadmonitor.yaml @@ -9,12 +9,12 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} spec: - replicas: {{ .Values.replicas }} - minReplicas: 1 + replicas: {{ .Values.cluster.processCounts.storage }} + minReplicas: {{ include "foundationdb.minReplicas" . }} kind: foundationdb type: foundationdb selector: - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/name: foundationdb + foundationdb.org/fdb-cluster-name: {{ .Release.Name }} + foundationdb.org/fdb-process-class: storage version: {{ .Chart.Version }} {{- end }} \ No newline at end of file diff --git a/packages/apps/foundationdb/values.schema.json b/packages/apps/foundationdb/values.schema.json index c1395a6a..1b0c6577 100644 --- a/packages/apps/foundationdb/values.schema.json +++ b/packages/apps/foundationdb/values.schema.json @@ -110,11 +110,15 @@ "stateless": -1, "storage": 3 }, + "redundancyMode": "double", + "storageEngine": "ssd-2", "version": "7.3.63" }, "required": [ "faultDomain", "processCounts", + "redundancyMode", + "storageEngine", "version" ], "properties": { @@ -173,6 +177,16 @@ } } }, + "redundancyMode": { + "description": "Database redundancy mode (single, double, triple, three_datacenter, three_datacenter_fallback)", + "type": "string", + "default": "double" + }, + "storageEngine": { + "description": "Storage engine (ssd-2, ssd-redwood-v1, ssd-rocksdb-v1, memory)", + "type": "string", + "default": "ssd-2" + }, "version": { "description": "Version of FoundationDB to use", "type": "string", diff --git a/packages/apps/foundationdb/values.yaml b/packages/apps/foundationdb/values.yaml index 4c5ccfbd..902f986a 100644 --- a/packages/apps/foundationdb/values.yaml +++ b/packages/apps/foundationdb/values.yaml @@ -10,6 +10,8 @@ ## @field clusterProcessCounts.storage {int} Number of storage processes (determines cluster size) ## @field clusterProcessCounts.cluster_controller {int} Number of cluster controller processes ## @field cluster.version {string} Version of FoundationDB to use +## @field cluster.redundancyMode {string} Database redundancy mode (single, double, triple, three_datacenter, three_datacenter_fallback) +## @field cluster.storageEngine {string} Storage engine (ssd-2, ssd-redwood-v1, ssd-rocksdb-v1, memory) ## @field cluster.faultDomain {clusterFaultDomain} Fault domain configuration ## @field clusterFaultDomain.key {string} Fault domain key ## @field clusterFaultDomain.valueFrom {string} Fault domain value source @@ -20,6 +22,8 @@ cluster: cluster_controller: 1 version: "7.3.63" + redundancyMode: "double" # Database redundancy mode + storageEngine: "ssd-2" # Storage engine faultDomain: key: "kubernetes.io/hostname"