[tests] cleanup state before repeat e2e-apps

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

.github/workflows/pre-commit.yml

@@ -29,6 +29,7 @@ jobs:
       - name: Install generate
         run: |
           curl -sSL https://github.com/cozystack/readme-generator-for-helm/releases/download/v1.0.0/readme-generator-for-helm-linux-amd64.tar.gz | tar -xzvf- -C /usr/local/bin/ readme-generator-for-helm
+          curl -sSL https://github.com/cozystack/cozyvalues-gen/releases/download/v0.7.0/cozyvalues-gen-linux-amd64.tar.gz | tar -xzvf- -C /usr/local/bin/ cozyvalues-gen
 
       - name: Run pre-commit hooks
         run: |

.github/workflows/tags.yaml

@@ -149,36 +149,35 @@ jobs:
           version:     ${{ steps.tag.outputs.tag }}            # A
           compare-to:  ${{ steps.latest_release.outputs.tag }} # B
 
-      # Create or reuse DRAFT GitHub Release
+      # Create or reuse draft release
       - name: Create / reuse draft release
         if: steps.check_release.outputs.skip == 'false'
         id: release
         uses: actions/github-script@v7
         with:
           script: |
-            const tag        = '${{ steps.tag.outputs.tag }}';
-            const isRc       = ${{ steps.tag.outputs.is_rc }};
-            const outdated   = '${{ steps.semver.outputs.comparison-result }}' === '<';
-            const makeLatest = outdated ? false : 'legacy';
-            const releases   = await github.rest.repos.listReleases({
+            const tag      = '${{ steps.tag.outputs.tag }}';
+            const isRc     = ${{ steps.tag.outputs.is_rc }};
+            const releases = await github.rest.repos.listReleases({
               owner: context.repo.owner,
               repo:  context.repo.repo
             });
-            let rel          = releases.data.find(r => r.tag_name === tag);
+
+            let rel = releases.data.find(r => r.tag_name === tag);
             if (!rel) {
               rel = await github.rest.repos.createRelease({
                 owner: context.repo.owner,
                 repo:  context.repo.repo,
-                tag_name:    tag,
-                name:        tag,
-                draft:       true,
-                prerelease:  isRc,
-                make_latest: makeLatest
+                tag_name:   tag,
+                name:       tag,
+                draft:      true,
+                prerelease: isRc               // no make_latest for drafts
               });
               console.log(`Draft release created for ${tag}`);
             } else {
               console.log(`Re-using existing release ${tag}`);
             }
+
             core.setOutput('upload_url', rel.upload_url);
 
       # Build + upload assets (optional)

.gitignore

@@ -77,3 +77,5 @@ fabric.properties
 
 .DS_Store
 **/.DS_Store
+
+tmp/

api/v1alpha1/cozystackresourcedefinitions_types.go

@@ -0,0 +1,89 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +kubebuilder:object:root=true
+
+// CozystackResourceDefinition is the Schema for the cozystackresourcedefinitions API
+type CozystackResourceDefinition struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec CozystackResourceDefinitionSpec `json:"spec,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// CozystackResourceDefinitionList contains a list of CozystackResourceDefinition
+type CozystackResourceDefinitionList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []CozystackResourceDefinition `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&CozystackResourceDefinition{}, &CozystackResourceDefinitionList{})
+}
+
+type CozystackResourceDefinitionSpec struct {
+	// Application configuration
+	Application CozystackResourceDefinitionApplication `json:"application"`
+	// Release configuration
+	Release CozystackResourceDefinitionRelease `json:"release"`
+}
+
+type CozystackResourceDefinitionChart struct {
+	// Name of the Helm chart
+	Name string `json:"name"`
+	// Source reference for the Helm chart
+	SourceRef SourceRef `json:"sourceRef"`
+}
+
+type SourceRef struct {
+	// Kind of the source reference
+	// +kubebuilder:default:="HelmRepository"
+	Kind string `json:"kind"`
+	// Name of the source reference
+	Name string `json:"name"`
+	// Namespace of the source reference
+	// +kubebuilder:default:="cozy-public"
+	Namespace string `json:"namespace"`
+}
+
+type CozystackResourceDefinitionApplication struct {
+	// Kind of the application, used for UI and API
+	Kind string `json:"kind"`
+	// OpenAPI schema for the application, used for API validation
+	OpenAPISchema string `json:"openAPISchema"`
+	// Plural name of the application, used for UI and API
+	Plural string `json:"plural"`
+	// Singular name of the application, used for UI and API
+	Singular string `json:"singular"`
+}
+
+type CozystackResourceDefinitionRelease struct {
+	// Helm chart configuration
+	Chart CozystackResourceDefinitionChart `json:"chart"`
+	// Labels for the release
+	Labels map[string]string `json:"labels,omitempty"`
+	// Prefix for the release name
+	Prefix string `json:"prefix"`
+}

api/v1alpha1/zz_generated.deepcopy.go

@@ -25,6 +25,135 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CozystackResourceDefinition) DeepCopyInto(out *CozystackResourceDefinition) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CozystackResourceDefinition.
+func (in *CozystackResourceDefinition) DeepCopy() *CozystackResourceDefinition {
+	if in == nil {
+		return nil
+	}
+	out := new(CozystackResourceDefinition)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CozystackResourceDefinition) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CozystackResourceDefinitionApplication) DeepCopyInto(out *CozystackResourceDefinitionApplication) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CozystackResourceDefinitionApplication.
+func (in *CozystackResourceDefinitionApplication) DeepCopy() *CozystackResourceDefinitionApplication {
+	if in == nil {
+		return nil
+	}
+	out := new(CozystackResourceDefinitionApplication)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CozystackResourceDefinitionChart) DeepCopyInto(out *CozystackResourceDefinitionChart) {
+	*out = *in
+	out.SourceRef = in.SourceRef
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CozystackResourceDefinitionChart.
+func (in *CozystackResourceDefinitionChart) DeepCopy() *CozystackResourceDefinitionChart {
+	if in == nil {
+		return nil
+	}
+	out := new(CozystackResourceDefinitionChart)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CozystackResourceDefinitionList) DeepCopyInto(out *CozystackResourceDefinitionList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]CozystackResourceDefinition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CozystackResourceDefinitionList.
+func (in *CozystackResourceDefinitionList) DeepCopy() *CozystackResourceDefinitionList {
+	if in == nil {
+		return nil
+	}
+	out := new(CozystackResourceDefinitionList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CozystackResourceDefinitionList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CozystackResourceDefinitionRelease) DeepCopyInto(out *CozystackResourceDefinitionRelease) {
+	*out = *in
+	out.Chart = in.Chart
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CozystackResourceDefinitionRelease.
+func (in *CozystackResourceDefinitionRelease) DeepCopy() *CozystackResourceDefinitionRelease {
+	if in == nil {
+		return nil
+	}
+	out := new(CozystackResourceDefinitionRelease)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CozystackResourceDefinitionSpec) DeepCopyInto(out *CozystackResourceDefinitionSpec) {
+	*out = *in
+	out.Application = in.Application
+	in.Release.DeepCopyInto(&out.Release)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CozystackResourceDefinitionSpec.
+func (in *CozystackResourceDefinitionSpec) DeepCopy() *CozystackResourceDefinitionSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(CozystackResourceDefinitionSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in Selector) DeepCopyInto(out *Selector) {
 	{
@@ -46,6 +175,21 @@ func (in Selector) DeepCopy() Selector {
 	return *out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SourceRef) DeepCopyInto(out *SourceRef) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SourceRef.
+func (in *SourceRef) DeepCopy() *SourceRef {
+	if in == nil {
+		return nil
+	}
+	out := new(SourceRef)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Workload) DeepCopyInto(out *Workload) {
 	*out = *in

hack/download-dashboards.sh

@@ -81,6 +81,7 @@ modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//main/capacity-p
 modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//flux/flux-control-plane.json
 modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//flux/flux-stats.json
 modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//kafka/strimzi-kafka.json
+modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//seaweedfs/seaweedfs.json
 modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//goldpinger/goldpinger.json
 EOT
 

hack/e2e-apps/bucket.bats

@@ -0,0 +1,48 @@
+#!/usr/bin/env bats
+
+@test "Create and Verify Seeweedfs Bucket" {
+  # Create the bucket resource
+  name='test'
+  kubectl -n tenant-test delete buckets.apps.cozystack.io "$name" --ignore-not-found
+  kubectl apply -f - <<EOF
+apiVersion: apps.cozystack.io/v1alpha1
+kind: Bucket
+metadata:
+  name: ${name}
+  namespace: tenant-test
+spec: {}
+EOF
+
+  # Wait for the bucket to be ready
+  kubectl -n tenant-test wait hr bucket-${name} --timeout=100s --for=condition=ready
+  kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=300s --for=jsonpath='{.status.bucketReady}'
+  kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name} --timeout=300s --for=jsonpath='{.status.accessGranted}'
+
+  # Get and decode credentials
+  kubectl -n tenant-test get secret bucket-${name} -ojsonpath='{.data.BucketInfo}' | base64 -d > bucket-test-credentials.json
+
+  # Get credentials from the secret
+  ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' bucket-test-credentials.json)
+  SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' bucket-test-credentials.json)
+  BUCKET_NAME=$(jq -r '.spec.bucketName' bucket-test-credentials.json)
+
+  # Start port-forwarding
+  bash -c 'timeout 100s kubectl port-forward service/seaweedfs-s3 -n tenant-root 8333:8333 > /dev/null 2>&1 &'
+
+  # Wait for port-forward to be ready
+  timeout 30 sh -ec 'until nc -z localhost 8333; do sleep 1; done'
+
+  # Set up MinIO alias with error handling
+  mc alias set local https://localhost:8333 $ACCESS_KEY $SECRET_KEY --insecure
+
+  # Upload file to bucket
+  mc cp bucket-test-credentials.json $BUCKET_NAME/bucket-test-credentials.json
+
+  # Verify file was uploaded
+  mc ls $BUCKET_NAME/bucket-test-credentials.json
+
+  # Clean up uploaded file
+  mc rm $BUCKET_NAME/bucket-test-credentials.json
+
+  kubectl -n tenant-test delete bucket.apps.cozystack.io ${name}
+}

hack/e2e-apps/clickhouse.bats

@@ -2,6 +2,7 @@
 
 @test "Create DB ClickHouse" {
   name='test'
+  kubectl -n tenant-test delete clickhouses.apps.cozystack.io $name --ignore-not-found
   kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: ClickHouse
@@ -27,6 +28,10 @@ spec:
     s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
     s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
     resticPassword: ChaXoveekoh6eigh4siesheeda2quai0
+  clickhouseKeeper:
+    enabled: true
+    resourcesPreset: "micro"
+    size: "1Gi"
   resources: {}
   resourcesPreset: "nano"
 EOF

hack/e2e-apps/kafka.bats

@@ -2,6 +2,7 @@
 
 @test "Create Kafka" {
   name='test'
+  kubectl -n tenant-test delete kafkas.apps.cozystack.io "$name" --ignore-not-found
   kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Kafka

hack/e2e-apps/mysql.bats

@@ -2,6 +2,7 @@
 
 @test "Create DB MySQL" {
   name='test'
+  kubectl -n tenant-test delete mysqls.apps.cozystack.io $name --ignore-not-found
   kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: MySQL

hack/e2e-apps/postgres.bats

@@ -2,6 +2,7 @@
 
 @test "Create DB PostgreSQL" {
   name='test'
+  kubectl -n tenant-test delete postgreses.apps.cozystack.io $name --ignore-not-found
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Postgres

hack/e2e-apps/redis.bats

@@ -2,6 +2,7 @@
 
 @test "Create Redis" {
   name='test'
+  kubectl -n tenant-test delete redises.apps.cozystack.io $name --ignore-not-found
   kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Redis

hack/e2e-apps/run-kubernetes.sh

@@ -4,6 +4,7 @@ run_kubernetes_test() {
     local port="$3"
     local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml)
 
+  kubectl -n tenant-test delete kuberneteses.apps.cozystack.io $test_name --ignore-not-found
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Kubernetes

hack/e2e-apps/virtualmachine.bats

@@ -2,6 +2,7 @@
 
 @test "Create a Virtual Machine" {
   name='test'
+  kubectl -n tenant-test delete virtualmachines.apps.cozystack.io $name --ignore-not-found
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VirtualMachine
@@ -20,9 +21,7 @@ spec:
     storage: 5Gi
     storageClass: replicated
   gpus: []
-  resources:
-    cpu: ""
-    memory: ""
+  resources: {}
   sshKeys:
   - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF
     test@test

hack/e2e-apps/vminstance.bats

@@ -1,5 +1,12 @@
 #!/usr/bin/env bats
 
+@test "Cleanup" {
+  name='test'
+  diskName='test'
+  kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName --ignore-not-found
+  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found
+}
+
 @test "Create a VM Disk" {
   name='test'
   kubectl apply -f - <<EOF

hack/e2e-install-cozystack.bats

@@ -123,10 +123,10 @@ EOF
 
 @test "Configure Tenant and wait for applications" {
   # Patch root tenant and wait for its releases
-  kubectl patch tenants/root -n tenant-root --type merge -p '{"spec":{"host":"example.org","ingress":true,"monitoring":true,"etcd":true,"isolated":true}}'
+  kubectl patch tenants/root -n tenant-root --type merge -p '{"spec":{"host":"example.org","ingress":true,"monitoring":true,"etcd":true,"isolated":true, "seaweedfs": true}}'
 
-  timeout 60 sh -ec 'until kubectl get hr -n tenant-root etcd ingress monitoring tenant-root >/dev/null 2>&1; do sleep 1; done'
-  kubectl wait hr/etcd hr/ingress hr/tenant-root -n tenant-root --timeout=2m --for=condition=ready
+  timeout 60 sh -ec 'until kubectl get hr -n tenant-root etcd ingress monitoring seaweedfs tenant-root >/dev/null 2>&1; do sleep 1; done'
+  kubectl wait hr/etcd hr/ingress hr/tenant-root hr/seaweedfs -n tenant-root --timeout=4m --for=condition=ready
 
   if ! kubectl wait hr/monitoring -n tenant-root --timeout=2m --for=condition=ready; then
     flux reconcile hr monitoring -n tenant-root --force

hack/e2e-prepare-cluster.bats

@@ -136,25 +136,25 @@ machine:
     mirrors:
       docker.io:
         endpoints:
-        - https://dockerio.nexus.lllamnyp.su
+        - https://dockerio.nexus.aenix.org
       cr.fluentbit.io:
         endpoints:
-        - https://fluentbit.nexus.lllamnyp.su
+        - https://fluentbit.nexus.aenix.org
       docker-registry3.mariadb.com:
         endpoints:
-        - https://mariadb.nexus.lllamnyp.su
+        - https://mariadb.nexus.aenix.org
       gcr.io:
         endpoints:
-        - https://gcr.nexus.lllamnyp.su
+        - https://gcr.nexus.aenix.org
       ghcr.io:
         endpoints:
-        - https://ghcr.nexus.lllamnyp.su
+        - https://ghcr.nexus.aenix.org
       quay.io:
         endpoints:
-        - https://quay.nexus.lllamnyp.su
+        - https://quay.nexus.aenix.org
       registry.k8s.io:
         endpoints:
-        - https://k8s.nexus.lllamnyp.su
+        - https://k8s.nexus.aenix.org
   files:
   - content: |
       [plugins]

hack/update-codegen.sh

@@ -32,6 +32,10 @@ kube::codegen::gen_helpers \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" \
     "${SCRIPT_ROOT}/pkg/apis"
 
+kube::codegen::gen_helpers \
+    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" \
+    "${SCRIPT_ROOT}/api"
+
 if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then
     report_filename="${API_KNOWN_VIOLATIONS_DIR}/cozystack_api_violation_exceptions.list"
     if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then

packages/apps/README.md

@@ -4,6 +4,5 @@
 cd packages/core/installer
 make image-cozystack REGISTRY=YOUR_CUSTOM_REGISTRY
 make apply
-kubectl delete pod dashboard-redis-master-0 -n cozy-dashboard
 kubectl delete po -l app=source-controller -n cozy-fluxcd
 ```

packages/apps/clickhouse/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.11.1
+version: 0.12.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/clickhouse/Makefile

@@ -7,6 +7,7 @@ include ../../../scripts/package.mk
 generate:
 	readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md
 	yq -i -o json --indent 4 '.properties.resourcesPreset.enum = $(PRESET_ENUM)' values.schema.json
+	yq -i -o json --indent 4 '.properties.clickhouseKeeper.resourcesPreset.enum = $(PRESET_ENUM)' values.schema.json
 
 image:
 	docker buildx build images/clickhouse-backup \

packages/apps/clickhouse/README.md

@@ -53,6 +53,15 @@ For more details, read [Restic: Effective Backup from Stdin](https://blog.aenix.
 | `backup.s3SecretKey`     | Secret key for S3, used for authentication     | `ju3eum4dekeich9ahM1te8waeGai0oog`                     |
 | `backup.resticPassword`  | Password for Restic backup encryption          | `ChaXoveekoh6eigh4siesheeda2quai0`                     |
 
+### clickhouseKeeper parameters
+
+| Name                               | Description                                                                                                                 | Value   |
+| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | ------- |
+| `clickhouseKeeper.enabled`         | Deploy ClickHouse Keeper for cluster coordination                                                                           | `true`  |
+| `clickhouseKeeper.size`            | Persistent Volume Claim size, available for application data                                                                | `1Gi`   |
+| `clickhouseKeeper.resourcesPreset` | Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge. | `micro` |
+| `clickhouseKeeper.replicas`        | Number of keeper replicas                                                                                                   | `3`     |
+
 ## Parameter examples and reference
 
 ### resources and resourcesPreset

packages/apps/clickhouse/templates/chkeeper.yaml

@@ -0,0 +1,96 @@
+{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
+{{- $clusterDomain := (index $cozyConfig.data "cluster-domain") | default "cozy.local" }}
+
+{{- if .Values.clickhouseKeeper.enabled }}
+apiVersion: "clickhouse-keeper.altinity.com/v1"
+kind: "ClickHouseKeeperInstallation"
+metadata:
+  name: "{{ .Release.Name }}-keeper"
+  annotations:
+    prometheus.io/port: "7000"
+    prometheus.io/scrape: "true"
+spec:
+  namespaceDomainPattern: "%s.svc.{{ $clusterDomain }}"
+  configuration:
+    clusters:
+      - name: "cluster1"
+        layout:
+          replicasCount: {{ .Values.clickhouseKeeper.replicas }}
+    settings:
+      logger/level: "trace"
+      logger/console: "true"
+      listen_host: "0.0.0.0"
+      keeper_server/four_letter_word_white_list: "*"
+      keeper_server/coordination_settings/raft_logs_level: "information"
+      prometheus/endpoint: "/metrics"
+      prometheus/port: "7000"
+      prometheus/metrics: "true"
+      prometheus/events: "true"
+      prometheus/asynchronous_metrics: "true"
+      prometheus/status_info: "false"
+
+  defaults:
+    templates:
+      # Templates are specified as default for all clusters
+      podTemplate: default
+      dataVolumeClaimTemplate: default
+
+  templates:
+    podTemplates:
+      - name: default
+        metadata:
+          labels:
+            app: "{{ .Release.Name }}-keeper"
+          annotations:
+            prometheus.io/port: "7000"
+            prometheus.io/scrape: "true"
+        spec:
+          affinity:
+            podAntiAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchExpressions:
+                      - key: "app"
+                        operator: In
+                        values:
+                          - "{{ .Release.Name }}-keeper"
+                  topologyKey: "kubernetes.io/hostname"
+          containers:
+            - name: clickhouse-keeper
+              imagePullPolicy: IfNotPresent
+              image: clickhouse/clickhouse-keeper:24.9.2.42
+              resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.clickhouseKeeper.resourcesPreset .Values.resources $) | nindent 20 }}  
+          securityContext:
+            fsGroup: 101
+
+    volumeClaimTemplates:
+      - name: default
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: "{{ .Values.clickhouseKeeper.size }}"
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: {{ .Release.Name }}-keeper
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-keeper
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scheme: http
+      relabelConfigs:
+        - action: replace
+          sourceLabels: [__meta_kubernetes_pod_node_name]
+          targetLabel: instance
+{{- end }}

packages/apps/clickhouse/templates/clickhouse.yaml

@@ -91,6 +91,18 @@ spec:
         layout:
           shardsCount: {{ .Values.shards }}
           replicasCount: {{ .Values.replicas }}
+    {{- if .Values.clickhouseKeeper.enabled }}
+    zookeeper:
+      nodes:
+        {{- $replicas := int .Values.clickhouseKeeper.replicas }}
+        {{- $release := .Release.Name }}
+        {{- $namespace := .Release.Namespace }}
+        {{- $clusterDomain := .Values.clusterDomain }}
+        {{- range $i := until $replicas }}
+        - host: "chk-{{ $release }}-keeper-cluster1-0-{{ $i }}.{{ $namespace }}.svc.{{ $clusterDomain }}"
+          port: 2181
+        {{- end }}
+    {{- end }}
   templates:
     volumeClaimTemplates:
       - name: data-volume-template

packages/apps/clickhouse/templates/dashboard-resourcemap.yaml

@@ -23,6 +23,9 @@ rules:
   - workloadmonitors
   resourceNames:
   - {{ .Release.Name }}
+  {{- if .Values.clickhouseKeeper.enabled }}
+  - {{ .Release.Name }}-keeper
+  {{- end }}
   verbs: ["get", "list", "watch"]
 ---
 kind: RoleBinding

packages/apps/clickhouse/templates/workloadmonitor.yaml

@@ -11,3 +11,18 @@ spec:
   selector:
     app.kubernetes.io/instance: {{ $.Release.Name }}
   version: {{ $.Chart.Version }}
+{{- if .Values.clickhouseKeeper.enabled }}
+---
+apiVersion: cozystack.io/v1alpha1
+kind: WorkloadMonitor
+metadata:
+  name: {{ $.Release.Name }}-keeper
+spec:
+  replicas: {{ .Values.clickhouseKeeper.replicas }}
+  minReplicas: 1
+  kind: clickhouse
+  type: clickhouse
+  selector:
+    app: {{ $.Release.Name }}-keeper
+  version: {{ $.Chart.Version }}
+{{- end }}

packages/apps/clickhouse/values.schema.json

@@ -45,6 +45,42 @@
             },
             "type": "object"
         },
+        "clickhouseKeeper": {
+            "properties": {
+                "enabled": {
+                    "default": true,
+                    "description": "Deploy ClickHouse Keeper for cluster coordination  ",
+                    "type": "boolean"
+                },
+                "replicas": {
+                    "default": 3,
+                    "description": "Number of keeper replicas",
+                    "type": "number"
+                },
+                "resourcesPreset": {
+                    "default": "micro",
+                    "description": "Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge.",
+                    "type": "string"
+                },
+                "size": {
+                    "default": "1Gi",
+                    "description": "Persistent Volume Claim size, available for application data",
+                    "type": "string"
+                }
+            },
+            "type": "object",
+            "resourcesPreset": {
+                "enum": [
+                    "nano",
+                    "micro",
+                    "small",
+                    "medium",
+                    "large",
+                    "xlarge",
+                    "2xlarge"
+                ]
+            }
+        },
         "logStorageSize": {
             "default": "2Gi",
             "description": "Size of Persistent Volume for logs",

packages/apps/clickhouse/values.yaml

@@ -56,3 +56,13 @@ backup:
   s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
   resticPassword: ChaXoveekoh6eigh4siesheeda2quai0
 
+## @section clickhouseKeeper parameters
+## @param clickhouseKeeper.enabled Deploy ClickHouse Keeper for cluster coordination  
+## @param clickhouseKeeper.size Persistent Volume Claim size, available for application data
+## @param clickhouseKeeper.resourcesPreset Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge.
+## @param clickhouseKeeper.replicas Number of keeper replicas
+clickhouseKeeper:
+  enabled: true
+  size: 1Gi
+  resourcesPreset: micro
+  replicas: 3

packages/apps/http-cache/images/nginx-cache.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/nginx-cache:0.6.1@sha256:e0a07082bb6fc6aeaae2315f335386f1705a646c72f9e0af512aebbca5cb2b15
+ghcr.io/cozystack/cozystack/nginx-cache:0.6.1@sha256:b7633717cd7449c0042ae92d8ca9b36e4d69566561f5c7d44e21058e7d05c6d5

packages/apps/kubernetes/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.26.0
+version: 0.26.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/kubernetes/images/cluster-autoscaler.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/cluster-autoscaler:0.26.0@sha256:3a8170433e1632e5cc2b6d9db34d0605e8e6c63c158282c38450415e700e932e
+ghcr.io/cozystack/cozystack/cluster-autoscaler:0.26.2@sha256:3a8170433e1632e5cc2b6d9db34d0605e8e6c63c158282c38450415e700e932e

packages/apps/kubernetes/images/kubevirt-cloud-provider.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:0.26.0@sha256:49843a0b670eab061627e48df338b2b8bc9f577dc2cfd4c2ed4071e02e64b424
+ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:0.26.2@sha256:5335c044313b69ee13b30ca4941687e509005e55f4ae25723861edbf2fbd6dd2

packages/apps/kubernetes/images/kubevirt-cloud-provider/Dockerfile

@@ -21,6 +21,6 @@ RUN go mod vendor
 
 RUN CGO_ENABLED=0 go build -mod=vendor -ldflags="-s -w" -o bin/kubevirt-cloud-controller-manager ./cmd/kubevirt-cloud-controller-manager
 
-FROM registry.access.redhat.com/ubi9/ubi-micro
+FROM scratch
 COPY --from=builder /go/src/kubevirt.io/cloud-provider-kubevirt/bin/kubevirt-cloud-controller-manager /bin/kubevirt-cloud-controller-manager
 ENTRYPOINT [ "/bin/kubevirt-cloud-controller-manager" ]

packages/apps/kubernetes/images/kubevirt-csi-driver.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.26.0@sha256:445c2727b04ac68595b43c988ff17b3d69a7b22b0644fde3b10c65b47a7bc036
+ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.26.2@sha256:761e7235ff9cb7f6f223f00954943e6a5af32ed6624ee592a8610122f96febb0

packages/apps/kubernetes/templates/helmreleases/csi.yaml

@@ -35,6 +35,8 @@ spec:
     storageClass: "{{ . }}"
   {{- end }}
   dependsOn:
+  - name: {{ .Release.Name }}-vsnap-crd
+    namespace: {{ .Release.Namespace }}
   {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
   - name: {{ .Release.Name }}
     namespace: {{ .Release.Namespace }}

packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml

@@ -3,6 +3,7 @@
 {{- $clusterDomain := (index $cozyConfig.data "cluster-domain") | default "cozy.local" }}
 {{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }}
 {{- $targetTenant := index $myNS.metadata.annotations "namespace.cozystack.io/monitoring" }}
+vpaForVPA: false
 vertical-pod-autoscaler:
   recommender:
     extraArgs:

packages/apps/kubernetes/templates/helmreleases/volumesnapshot_crd.yaml

@@ -1,16 +1,16 @@
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: {{ .Release.Name }}-volumesnapshot-crd-for-tenant-k8s
+  name: {{ .Release.Name }}-vsnap-crd
   labels:
     cozystack.io/repository: system
     cozystack.io/target-cluster-name: {{ .Release.Name }}
 spec:
   interval: 5m
-  releaseName: volumesnapshot-crd-for-tenant-k8s
+  releaseName: vsnap-crd
   chart:
     spec:
-      chart: cozy-volumesnapshot-crd-for-tenant-k8s
+      chart: cozy-vsnap-crd
       reconcileStrategy: Revision
       sourceRef:
         kind: HelmRepository
@@ -21,8 +21,8 @@ spec:
     secretRef:
       name: {{ .Release.Name }}-admin-kubeconfig
       key: super-admin.svc
-  targetNamespace: cozy-volumesnapshot-crd-for-tenant-k8s
-  storageNamespace: cozy-volumesnapshot-crd-for-tenant-k8s
+  targetNamespace: cozy-vsnap-crd
+  storageNamespace: cozy-vsnap-crd
   install:
     createNamespace: true
     remediation:

packages/apps/postgres/Makefile

@@ -1,6 +1,4 @@
 include ../../../scripts/package.mk
-PRESET_ENUM := ["nano","micro","small","medium","large","xlarge","2xlarge"]
 
 generate:
-	readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md
-	yq -i -o json --indent 4 '.properties.resourcesPreset.enum = $(PRESET_ENUM)' values.schema.json
+	cozyvalues-gen -v values.yaml -s values.schema.json -r README.md

packages/apps/postgres/README.md

@@ -66,44 +66,61 @@ See:
 
 ### Common parameters
 
-| Name              | Description                                                                                                                             | Value   |
-| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ------- |
-| `replicas`        | Number of Postgres replicas                                                                                                             | `2`     |
-| `resources`       | Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied. | `{}`    |
-| `resourcesPreset` | Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge.             | `micro` |
-| `size`            | Persistent Volume size                                                                                                                  | `10Gi`  |
-| `storageClass`    | StorageClass used to store the data                                                                                                     | `""`    |
-| `external`        | Enable external access from outside the cluster                                                                                         | `false` |
+| Name               | Description                                                                                                                               | Type      | Value   |
+| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------- |
+| `replicas`         | Number of Postgres replicas                                                                                                               | `int`     | `2`     |
+| `resources`        | Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied.   | `*object` | `{}`    |
+| `resources.cpu`    | CPU                                                                                                                                       | `*string` | `null`  |
+| `resources.memory` | Memory                                                                                                                                    | `*string` | `null`  |
+| `resourcesPreset`  | Default sizing preset used when `resources` is omitted. Allowed values: `nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`. | `string`  | `{}`    |
+| `size`             | Persistent Volume Claim size, available for application data                                                                              | `string`  | `10Gi`  |
+| `storageClass`     | StorageClass used to store the data                                                                                                       | `string`  | `""`    |
+| `external`         | Enable external access from outside the cluster                                                                                           | `bool`    | `false` |
+
 
 ### Application-specific parameters
 
-| Name                                    | Description                                                                                                              | Value |
-| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ----- |
-| `postgresql.parameters.max_connections` | Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections | `100` |
-| `quorum.minSyncReplicas`                | Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.            | `0`   |
-| `quorum.maxSyncReplicas`                | Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).  | `0`   |
-| `users`                                 | Users configuration                                                                                                      | `{}`  |
-| `databases`                             | Databases configuration                                                                                                  | `{}`  |
+| Name                                    | Description                                                                                                              | Type                | Value   |
+| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ------------------- | ------- |
+| `postgresql`                            | PostgreSQL server configuration                                                                                          | `object`            | `{}`    |
+| `postgresql.parameters`                 | PostgreSQL server parameters                                                                                             | `object`            | `{}`    |
+| `postgresql.parameters.max_connections` | Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections | `int`               | `100`   |
+| `quorum`                                | Quorum configuration for synchronous replication                                                                         | `object`            | `{}`    |
+| `quorum.minSyncReplicas`                | Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.            | `int`               | `0`     |
+| `quorum.maxSyncReplicas`                | Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).  | `int`               | `0`     |
+| `users`                                 | Users configuration                                                                                                      | `map[string]object` | `{...}` |
+| `users[name].password`                  | Password for the user                                                                                                    | `*string`           | `null`  |
+| `users[name].replication`               | Whether the user has replication privileges                                                                              | `*bool`             | `null`  |
+| `databases`                             | Databases configuration                                                                                                  | `map[string]object` | `{...}` |
+| `databases[name].roles`                 | Roles for the database                                                                                                   | `*object`           | `null`  |
+| `databases[name].roles.admin`           | List of users with admin privileges                                                                                      | `[]string`          | `[]`    |
+| `databases[name].roles.readonly`        | List of users with read-only privileges                                                                                  | `[]string`          | `[]`    |
+| `databases[name].extensions`            | Extensions enabled for the database                                                                                      | `[]string`          | `[]`    |
+
 
 ### Backup parameters
 
-| Name                     | Description                                                | Value                               |
-| ------------------------ | ---------------------------------------------------------- | ----------------------------------- |
-| `backup.enabled`         | Enable regular backups                                     | `false`                             |
-| `backup.schedule`        | Cron schedule for automated backups                        | `0 2 * * * *`                       |
-| `backup.retentionPolicy` | Retention policy                                           | `30d`                               |
-| `backup.destinationPath` | Path to store the backup (i.e. s3://bucket/path/to/folder) | `s3://bucket/path/to/folder/`       |
-| `backup.endpointURL`     | S3 Endpoint used to upload data to the cloud               | `http://minio-gateway-service:9000` |
-| `backup.s3AccessKey`     | Access key for S3, used for authentication                 | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu`  |
-| `backup.s3SecretKey`     | Secret key for S3, used for authentication                 | `ju3eum4dekeich9ahM1te8waeGai0oog`  |
+| Name                     | Description                                                | Type     | Value                               |
+| ------------------------ | ---------------------------------------------------------- | -------- | ----------------------------------- |
+| `backup`                 | Backup configuration                                       | `object` | `{}`                                |
+| `backup.enabled`         | Enable regular backups                                     | `bool`   | `false`                             |
+| `backup.schedule`        | Cron schedule for automated backups                        | `string` | `0 2 * * * *`                       |
+| `backup.retentionPolicy` | Retention policy                                           | `string` | `30d`                               |
+| `backup.destinationPath` | Path to store the backup (i.e. s3://bucket/path/to/folder) | `string` | `s3://bucket/path/to/folder/`       |
+| `backup.endpointURL`     | S3 Endpoint used to upload data to the cloud               | `string` | `http://minio-gateway-service:9000` |
+| `backup.s3AccessKey`     | Access key for S3, used for authentication                 | `string` | `<access key>`                      |
+| `backup.s3SecretKey`     | Secret key for S3, used for authentication                 | `string` | `<secret key>`                      |
+
 
 ### Bootstrap (recovery) parameters
 
-| Name                     | Description                                                                                                          | Value   |
-| ------------------------ | -------------------------------------------------------------------------------------------------------------------- | ------- |
-| `bootstrap.enabled`      | Restore database cluster from a backup                                                                               | `false` |
-| `bootstrap.recoveryTime` | Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest | `""`    |
-| `bootstrap.oldName`      | Name of database cluster before deleting                                                                             | `""`    |
+| Name                     | Description                                                                                                          | Type      | Value   |
+| ------------------------ | -------------------------------------------------------------------------------------------------------------------- | --------- | ------- |
+| `bootstrap`              | Bootstrap configuration                                                                                              | `object`  | `{}`    |
+| `bootstrap.enabled`      | Restore database cluster from a backup                                                                               | `bool`    | `false` |
+| `bootstrap.recoveryTime` | Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest | `*string` | `""`    |
+| `bootstrap.oldName`      | Name of database cluster before deleting                                                                             | `string`  | `""`    |
+
 
 ## Parameter examples and reference
 

packages/apps/postgres/values.schema.json

@@ -1,140 +1,257 @@
 {
-    "properties": {
-        "backup": {
-            "properties": {
-                "destinationPath": {
-                    "default": "s3://bucket/path/to/folder/",
-                    "description": "Path to store the backup (i.e. s3://bucket/path/to/folder)",
-                    "type": "string"
-                },
-                "enabled": {
-                    "default": false,
-                    "description": "Enable regular backups",
-                    "type": "boolean"
-                },
-                "endpointURL": {
-                    "default": "http://minio-gateway-service:9000",
-                    "description": "S3 Endpoint used to upload data to the cloud",
-                    "type": "string"
-                },
-                "retentionPolicy": {
-                    "default": "30d",
-                    "description": "Retention policy",
-                    "type": "string"
-                },
-                "s3AccessKey": {
-                    "default": "oobaiRus9pah8PhohL1ThaeTa4UVa7gu",
-                    "description": "Access key for S3, used for authentication",
-                    "type": "string"
-                },
-                "s3SecretKey": {
-                    "default": "ju3eum4dekeich9ahM1te8waeGai0oog",
-                    "description": "Secret key for S3, used for authentication",
-                    "type": "string"
-                },
-                "schedule": {
-                    "default": "0 2 * * * *",
-                    "description": "Cron schedule for automated backups",
-                    "type": "string"
-                }
-            },
-            "type": "object"
+  "title": "Chart Values",
+  "type": "object",
+  "properties": {
+    "backup": {
+      "description": "Backup configuration",
+      "type": "object",
+      "default": {
+        "destinationPath": "s3://bucket/path/to/folder/",
+        "enabled": false,
+        "endpointURL": "http://minio-gateway-service:9000",
+        "retentionPolicy": "30d",
+        "s3AccessKey": "\u003caccess key\u003e",
+        "s3SecretKey": "\u003csecret key\u003e",
+        "schedule": "0 2 * * * *"
+      },
+      "required": [
+        "destinationPath",
+        "enabled",
+        "endpointURL",
+        "retentionPolicy",
+        "s3AccessKey",
+        "s3SecretKey",
+        "schedule"
+      ],
+      "properties": {
+        "destinationPath": {
+          "description": "Path to store the backup (i.e. s3://bucket/path/to/folder)",
+          "type": "string",
+          "default": "s3://bucket/path/to/folder/"
         },
-        "bootstrap": {
-            "properties": {
-                "enabled": {
-                    "default": false,
-                    "description": "Restore database cluster from a backup",
-                    "type": "boolean"
-                },
-                "oldName": {
-                    "default": "",
-                    "description": "Name of database cluster before deleting",
-                    "type": "string"
-                },
-                "recoveryTime": {
-                    "default": "",
-                    "description": "Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest",
-                    "type": "string"
-                }
-            },
-            "type": "object"
+        "enabled": {
+          "description": "Enable regular backups",
+          "type": "boolean",
+          "default": false
         },
-        "databases": {
-            "default": {},
-            "description": "Databases configuration",
-            "type": "object"
+        "endpointURL": {
+          "description": "S3 Endpoint used to upload data to the cloud",
+          "type": "string",
+          "default": "http://minio-gateway-service:9000"
         },
-        "external": {
-            "default": false,
-            "description": "Enable external access from outside the cluster",
-            "type": "boolean"
+        "retentionPolicy": {
+          "description": "Retention policy",
+          "type": "string",
+          "default": "30d"
         },
-        "postgresql": {
-            "properties": {
-                "parameters": {
-                    "properties": {
-                        "max_connections": {
-                            "default": 100,
-                            "description": "Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections",
-                            "type": "number"
-                        }
-                    },
-                    "type": "object"
-                }
-            },
-            "type": "object"
+        "s3AccessKey": {
+          "description": "Access key for S3, used for authentication",
+          "type": "string",
+          "default": "\u003caccess key\u003e"
         },
-        "quorum": {
-            "properties": {
-                "maxSyncReplicas": {
-                    "default": 0,
-                    "description": "Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).",
-                    "type": "number"
-                },
-                "minSyncReplicas": {
-                    "default": 0,
-                    "description": "Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.",
-                    "type": "number"
-                }
-            },
-            "type": "object"
+        "s3SecretKey": {
+          "description": "Secret key for S3, used for authentication",
+          "type": "string",
+          "default": "\u003csecret key\u003e"
         },
-        "replicas": {
-            "default": 2,
-            "description": "Number of Postgres replicas",
-            "type": "number"
-        },
-        "resources": {
-            "default": {},
-            "description": "Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied.",
-            "type": "object"
-        },
-        "resourcesPreset": {
-            "default": "micro",
-            "description": "Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge.",
-            "type": "string",
-            "enum": [
-                "nano",
-                "micro",
-                "small",
-                "medium",
-                "large",
-                "xlarge",
-                "2xlarge"
-            ]
-        },
-        "size": {
-            "default": "10Gi",
-            "description": "Persistent Volume size",
-            "type": "string"
-        },
-        "storageClass": {
-            "default": "",
-            "description": "StorageClass used to store the data",
-            "type": "string"
+        "schedule": {
+          "description": "Cron schedule for automated backups",
+          "type": "string",
+          "default": "0 2 * * * *"
         }
+      }
     },
-    "title": "Chart Values",
-    "type": "object"
-}
+    "bootstrap": {
+      "description": "Bootstrap configuration",
+      "type": "object",
+      "default": {
+        "enabled": false,
+        "oldName": "",
+        "recoveryTime": ""
+      },
+      "required": [
+        "enabled",
+        "oldName"
+      ],
+      "properties": {
+        "enabled": {
+          "description": "Restore database cluster from a backup",
+          "type": "boolean",
+          "default": false
+        },
+        "oldName": {
+          "description": "Name of database cluster before deleting",
+          "type": "string"
+        },
+        "recoveryTime": {
+          "description": "Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest",
+          "type": "string"
+        }
+      }
+    },
+    "databases": {
+      "description": "Databases configuration",
+      "type": "object",
+      "default": {},
+      "additionalProperties": {
+        "type": "object",
+        "properties": {
+          "extensions": {
+            "description": "Extensions enabled for the database",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "roles": {
+            "description": "Roles for the database",
+            "type": "object",
+            "properties": {
+              "admin": {
+                "description": "List of users with admin privileges",
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "readonly": {
+                "description": "List of users with read-only privileges",
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "external": {
+      "description": "Enable external access from outside the cluster",
+      "type": "boolean",
+      "default": false
+    },
+    "postgresql": {
+      "description": "PostgreSQL server configuration",
+      "type": "object",
+      "default": {
+        "parameters": {
+          "max_connections": 100
+        }
+      },
+      "required": [
+        "parameters"
+      ],
+      "properties": {
+        "parameters": {
+          "description": "PostgreSQL server parameters",
+          "type": "object",
+          "default": {
+            "max_connections": 100
+          },
+          "required": [
+            "max_connections"
+          ],
+          "properties": {
+            "max_connections": {
+              "description": "Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections",
+              "type": "integer",
+              "default": 100
+            }
+          }
+        }
+      }
+    },
+    "quorum": {
+      "description": "Quorum configuration for synchronous replication",
+      "type": "object",
+      "default": {
+        "maxSyncReplicas": 0,
+        "minSyncReplicas": 0
+      },
+      "required": [
+        "maxSyncReplicas",
+        "minSyncReplicas"
+      ],
+      "properties": {
+        "maxSyncReplicas": {
+          "description": "Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).",
+          "type": "integer",
+          "default": 0
+        },
+        "minSyncReplicas": {
+          "description": "Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.",
+          "type": "integer",
+          "default": 0
+        }
+      }
+    },
+    "replicas": {
+      "description": "Number of Postgres replicas",
+      "type": "integer",
+      "default": 2
+    },
+    "resources": {
+      "description": "Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied.",
+      "type": "object",
+      "default": {},
+      "properties": {
+        "cpu": {
+          "description": "CPU",
+          "type": "string",
+          "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+          "x-kubernetes-int-or-string": true
+        },
+        "memory": {
+          "description": "Memory",
+          "type": "string",
+          "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+          "x-kubernetes-int-or-string": true
+        }
+      }
+    },
+    "resourcesPreset": {
+      "description": "Default sizing preset used when `resources` is omitted. Allowed values: `nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`.",
+      "type": "string",
+      "default": "micro",
+      "enum": [
+        "nano",
+        "micro",
+        "small",
+        "medium",
+        "large",
+        "xlarge",
+        "2xlarge"
+      ]
+    },
+    "size": {
+      "description": "Persistent Volume Claim size, available for application data",
+      "type": "string",
+      "default": "10Gi",
+      "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+      "x-kubernetes-int-or-string": true
+    },
+    "storageClass": {
+      "description": "StorageClass used to store the data",
+      "type": "string"
+    },
+    "users": {
+      "description": "Users configuration",
+      "type": "object",
+      "default": {},
+      "additionalProperties": {
+        "type": "object",
+        "properties": {
+          "password": {
+            "description": "Password for the user",
+            "type": "string"
+          },
+          "replication": {
+            "description": "Whether the user has replication privileges",
+            "type": "boolean"
+          }
+        }
+      }
+    }
+  }
+}

packages/apps/postgres/values.yaml

@@ -1,36 +1,44 @@
 ## @section Common parameters
 ##
-## @param replicas Number of Postgres replicas
+## @param replicas {int} Number of Postgres replicas
 replicas: 2
-## @param resources Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied.
+## @param resources {*resources} Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied.
+## @field resources.cpu {*quantity} CPU
+## @field resources.memory {*quantity} Memory
 resources: {}
   # resources:
   #   cpu: 4000m
   #   memory: 4Gi
-## @param resourcesPreset Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge.
+## @param resourcesPreset {string enum:"nano,micro,small,medium,large,xlarge,2xlarge"} Default sizing preset used when `resources` is omitted. Allowed values: `nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`.
 resourcesPreset: "micro"
-## @param size Persistent Volume size
+## @param size {quantity} Persistent Volume Claim size, available for application data
 size: 10Gi
-## @param storageClass StorageClass used to store the data
+## @param storageClass {string} StorageClass used to store the data
 storageClass: ""
-## @param external Enable external access from outside the cluster
+## @param external {bool} Enable external access from outside the cluster
 external: false
 
-
 ## @section Application-specific parameters
+## @param postgresql {postgresql} PostgreSQL server configuration
+## @field postgresql.parameters {postgresqlParameters} PostgreSQL server parameters
+## @field postgresqlParameters.max_connections {int} Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections
 ##
-## @param postgresql.parameters.max_connections Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections
 postgresql:
   parameters:
     max_connections: 100
 
-## @param quorum.minSyncReplicas Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.
-## @param quorum.maxSyncReplicas Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).
+## Configuration for the quorum-based synchronous replication
+## @param quorum {quorum} Quorum configuration for synchronous replication
+## @field quorum.minSyncReplicas {int} Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.
+## @field quorum.maxSyncReplicas {int} Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).
 quorum:
   minSyncReplicas: 0
   maxSyncReplicas: 0
 
-## @param users [object] Users configuration
+## @param users {map[string]user} Users configuration
+## @field user.password {*string} Password for the user
+## @field user.replication {*bool} Whether the user has replication privileges
+##
 ## Example:
 ## users:
 ##   user1:
@@ -44,7 +52,12 @@ quorum:
 ##
 users: {}
 
-## @param databases Databases configuration
+## @param databases {map[string]database} Databases configuration
+## @field database.roles {*databaseRoles} Roles for the database
+## @field databaseRoles.admin {[]string} List of users with admin privileges
+## @field databaseRoles.readonly {[]string} List of users with read-only privileges
+## @field database.extensions {[]string} Extensions enabled for the database
+##
 ## Example:
 ## databases:
 ##   myapp:
@@ -64,27 +77,29 @@ databases: {}
 
 ## @section Backup parameters
 
-## @param backup.enabled Enable regular backups
-## @param backup.schedule Cron schedule for automated backups
-## @param backup.retentionPolicy Retention policy
-## @param backup.destinationPath Path to store the backup (i.e. s3://bucket/path/to/folder)
-## @param backup.endpointURL S3 Endpoint used to upload data to the cloud
-## @param backup.s3AccessKey Access key for S3, used for authentication
-## @param backup.s3SecretKey Secret key for S3, used for authentication
+## @param backup {backup} Backup configuration
+## @field backup.enabled {bool} Enable regular backups
+## @field backup.schedule {string} Cron schedule for automated backups
+## @field backup.retentionPolicy {string} Retention policy
+## @field backup.destinationPath {string} Path to store the backup (i.e. s3://bucket/path/to/folder)
+## @field backup.endpointURL {string} S3 Endpoint used to upload data to the cloud
+## @field backup.s3AccessKey {string} Access key for S3, used for authentication
+## @field backup.s3SecretKey {string} Secret key for S3, used for authentication
 backup:
   enabled: false
   retentionPolicy: 30d
-  destinationPath: s3://bucket/path/to/folder/
-  endpointURL: http://minio-gateway-service:9000
+  destinationPath: "s3://bucket/path/to/folder/"
+  endpointURL: "http://minio-gateway-service:9000"
   schedule: "0 2 * * * *"
-  s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
-  s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
+  s3AccessKey: "<access key>"
+  s3SecretKey: "<secret key>"
 
 ## @section Bootstrap (recovery) parameters
 
-## @param bootstrap.enabled Restore database cluster from a backup
-## @param bootstrap.recoveryTime Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest
-## @param bootstrap.oldName Name of database cluster before deleting
+## @param bootstrap {bootstrap} Bootstrap configuration
+## @field bootstrap.enabled {bool} Restore database cluster from a backup
+## @field bootstrap.recoveryTime {*string} Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest
+## @field bootstrap.oldName {string} Name of database cluster before deleting
 ##
 bootstrap:
   enabled: false

packages/apps/tenant/Makefile

@@ -1,4 +1,4 @@
 include ../../../scripts/package.mk
 
 generate:
-	readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md
+	cozyvalues-gen -v values.yaml -s values.schema.json -r README.md

packages/apps/tenant/README.md

@@ -69,12 +69,13 @@ tenant-u1
 
 ### Common parameters
 
-| Name             | Description                                                                                                                 | Value   |
-| ---------------- | --------------------------------------------------------------------------------------------------------------------------- | ------- |
-| `host`           | The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host). | `""`    |
-| `etcd`           | Deploy own Etcd cluster                                                                                                     | `false` |
-| `monitoring`     | Deploy own Monitoring Stack                                                                                                 | `false` |
-| `ingress`        | Deploy own Ingress Controller                                                                                               | `false` |
-| `seaweedfs`      | Deploy own SeaweedFS                                                                                                        | `false` |
-| `isolated`       | Enforce tenant namespace with network policies                                                                              | `true`  |
-| `resourceQuotas` | Define resource quotas for the tenant                                                                                       | `{}`    |
+| Name             | Description                                                                                                                 | Type      | Value   |
+| ---------------- | --------------------------------------------------------------------------------------------------------------------------- | --------- | ------- |
+| `host`           | The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host). | `*string` | `""`    |
+| `etcd`           | Deploy own Etcd cluster                                                                                                     | `bool`    | `false` |
+| `monitoring`     | Deploy own Monitoring Stack                                                                                                 | `bool`    | `false` |
+| `ingress`        | Deploy own Ingress Controller                                                                                               | `bool`    | `false` |
+| `seaweedfs`      | Deploy own SeaweedFS                                                                                                        | `bool`    | `false` |
+| `isolated`       | Enforce tenant namespace with network policies, `true` by default                                                           | `bool`    | `true`  |
+| `resourceQuotas` | Define resource quotas for the tenant                                                                                       | `string`  | `{}`    |
+

packages/apps/tenant/templates/monitoring.yaml

@@ -17,6 +17,12 @@ spec:
         kind: HelmRepository
         name: cozystack-extra
         namespace: cozy-public
+  install:
+    remediation:
+      retries: 10
+  upgrade:
+    remediation:
+      retries: 10
   interval: 1m0s
-  timeout: 5m0s
+  timeout: 10m0s
 {{- end }}

packages/apps/tenant/values.schema.json

@@ -1,41 +1,45 @@
 {
-    "properties": {
-        "etcd": {
-            "default": false,
-            "description": "Deploy own Etcd cluster",
-            "type": "boolean"
-        },
-        "host": {
-            "default": "",
-            "description": "The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host).",
-            "type": "string"
-        },
-        "ingress": {
-            "default": false,
-            "description": "Deploy own Ingress Controller",
-            "type": "boolean"
-        },
-        "isolated": {
-            "default": true,
-            "description": "Enforce tenant namespace with network policies",
-            "type": "boolean"
-        },
-        "monitoring": {
-            "default": false,
-            "description": "Deploy own Monitoring Stack",
-            "type": "boolean"
-        },
-        "resourceQuotas": {
-            "default": {},
-            "description": "Define resource quotas for the tenant",
-            "type": "object"
-        },
-        "seaweedfs": {
-            "default": false,
-            "description": "Deploy own SeaweedFS",
-            "type": "boolean"
-        }
+  "title": "Chart Values",
+  "type": "object",
+  "properties": {
+    "etcd": {
+      "description": "Deploy own Etcd cluster",
+      "type": "boolean",
+      "default": false
     },
-    "title": "Chart Values",
-    "type": "object"
+    "host": {
+      "description": "The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host).",
+      "type": "string"
+    },
+    "ingress": {
+      "description": "Deploy own Ingress Controller",
+      "type": "boolean",
+      "default": false
+    },
+    "isolated": {
+      "description": "Enforce tenant namespace with network policies, `true` by default",
+      "type": "boolean",
+      "default": true
+    },
+    "monitoring": {
+      "description": "Deploy own Monitoring Stack",
+      "type": "boolean",
+      "default": false
+    },
+    "resourceQuotas": {
+      "description": "Define resource quotas for the tenant",
+      "type": "object",
+      "default": {},
+      "additionalProperties": {
+        "type": "string",
+        "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+        "x-kubernetes-int-or-string": true
+      }
+    },
+    "seaweedfs": {
+      "description": "Deploy own SeaweedFS",
+      "type": "boolean",
+      "default": false
+    }
+  }
 }

packages/apps/tenant/values.yaml

@@ -1,18 +1,18 @@
 ## @section Common parameters
 
-## @param host The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host).
-## @param etcd Deploy own Etcd cluster
-## @param monitoring Deploy own Monitoring Stack
-## @param ingress Deploy own Ingress Controller
-## @param seaweedfs Deploy own SeaweedFS
-## @param isolated Enforce tenant namespace with network policies
-## @param resourceQuotas Define resource quotas for the tenant
+## @param host {*string} The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host).
+## @param etcd {bool} Deploy own Etcd cluster
+## @param monitoring {bool} Deploy own Monitoring Stack
+## @param ingress {bool} Deploy own Ingress Controller
+## @param seaweedfs {bool} Deploy own SeaweedFS
+## @param isolated {bool} Enforce tenant namespace with network policies, `true` by default
 host: ""
 etcd: false
 monitoring: false
 ingress: false
 seaweedfs: false
 isolated: true
+## @param resourceQuotas {map[string]quantity} Define resource quotas for the tenant
 resourceQuotas: {}
 # resourceQuotas:
 #   cpu: "1"

packages/apps/versions_map

@@ -15,7 +15,8 @@ clickhouse 0.9.2 632224a3
 clickhouse 0.10.0 6358fd7a
 clickhouse 0.10.1 4369b031
 clickhouse 0.11.0 08cb7c0f
-clickhouse 0.11.1 HEAD
+clickhouse 0.11.1 0e47e1e8
+clickhouse 0.12.0 HEAD
 ferretdb 0.1.0 e9716091
 ferretdb 0.1.1 91b0499a
 ferretdb 0.2.0 6c5cf5bf
@@ -61,7 +62,10 @@ kubernetes 0.24.0 62cb694d
 kubernetes 0.25.0 70f82667
 kubernetes 0.25.1 acd4663a
 kubernetes 0.25.2 08cb7c0f
-kubernetes 0.26.0 HEAD
+kubernetes 0.26.0 9584e5f5
+kubernetes 0.26.1 0e47e1e8
+kubernetes 0.26.2 8ddbe32e
+kubernetes 0.26.3 HEAD
 mysql 0.1.0 263e47be
 mysql 0.2.0 c24a103f
 mysql 0.3.0 53f2365e
@@ -172,7 +176,8 @@ virtual-machine 0.10.2 632224a3
 virtual-machine 0.11.0 4369b031
 virtual-machine 0.12.0 acd4663a
 virtual-machine 0.12.1 909208ba
-virtual-machine 0.12.2 HEAD
+virtual-machine 0.12.2 8ddbe32e
+virtual-machine 0.12.3 HEAD
 vm-disk 0.1.0 d971f2ff
 vm-disk 0.1.1 6130f43d
 vm-disk 0.1.2 632224a3
@@ -191,7 +196,8 @@ vm-instance 0.7.2 632224a3
 vm-instance 0.8.0 4369b031
 vm-instance 0.9.0 acd4663a
 vm-instance 0.10.0 909208ba
-vm-instance 0.10.1 HEAD
+vm-instance 0.10.1 8ddbe32e
+vm-instance 0.10.2 HEAD
 vpn 0.1.0 263e47be
 vpn 0.2.0 53f2365e
 vpn 0.3.0 6c5cf5bf

packages/apps/virtual-machine/Chart.yaml

@@ -17,7 +17,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.12.2
+version: 0.12.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/virtual-machine/Makefile

@@ -1,12 +1,9 @@
 include ../../../scripts/package.mk
 
 generate:
-	readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md
+	cozyvalues-gen -v values.yaml -s values.schema.json -r README.md
 	yq -o json -i '.properties.gpus.items.type = "object" | .properties.gpus.default = []' values.schema.json
-	INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \
-	  && yq -i -o json ".properties.instanceType.optional=true | .properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json
+	# INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \
+	#  && yq -i -o json ".properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json
 	PREFERENCES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/preferences.yaml | yq 'split(" ") | . + [""]' -o json) \
-	  && yq -i -o json ".properties.instanceProfile.optional=true | .properties.instanceProfile.enum = $${PREFERENCES}" values.schema.json
-	yq -i -o json '.properties.externalPorts.items.type = "integer"' values.schema.json
-	yq -i -o json '.properties.systemDisk.properties.image.enum = ["ubuntu", "cirros", "alpine", "fedora", "talos"]' values.schema.json
-	yq -i -o json '.properties.externalMethod.enum = ["PortList", "WholeIP"]' values.schema.json
+	  && yq -i -o json ".properties.instanceProfile.enum = $${PREFERENCES}" values.schema.json

packages/apps/virtual-machine/README.md

@@ -36,24 +36,28 @@ virtctl ssh <user>@<vm>
 
 ### Common parameters
 
-| Name                      | Description                                                                                                | Value        |
-| ------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------ |
-| `external`                | Enable external access from outside the cluster                                                            | `false`      |
-| `externalMethod`          | specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` | `PortList`   |
-| `externalPorts`           | Specify ports to forward from outside the cluster                                                          | `[]`         |
-| `running`                 | Determines if the virtual machine should be running                                                        | `true`       |
-| `instanceType`            | Virtual Machine instance type                                                                              | `u1.medium`  |
-| `instanceProfile`         | Virtual Machine preferences profile                                                                        | `ubuntu`     |
-| `systemDisk.image`        | The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos` | `ubuntu`     |
-| `systemDisk.storage`      | The size of the disk allocated for the virtual machine                                                     | `5Gi`        |
-| `systemDisk.storageClass` | StorageClass used to store the data                                                                        | `replicated` |
-| `gpus`                    | List of GPUs to attach                                                                                     | `[]`         |
-| `resources.cpu`           | The number of CPU cores allocated to the virtual machine                                                   | `""`         |
-| `resources.memory`        | The amount of memory allocated to the virtual machine                                                      | `""`         |
-| `resources.sockets`       | The number of CPU sockets allocated to the virtual machine (used to define vCPU topology)                  | `""`         |
-| `sshKeys`                 | List of SSH public keys for authentication. Can be a single key or a list of keys.                         | `[]`         |
-| `cloudInit`               | cloud-init user data config. See cloud-init documentation for more details.                                | `""`         |
-| `cloudInitSeed`           | A seed string to generate an SMBIOS UUID for the VM.                                                       | `""`         |
+| Name                      | Description                                                                                                 | Type       | Value        |
+| ------------------------- | ----------------------------------------------------------------------------------------------------------- | ---------- | ------------ |
+| `external`                | Enable external access from outside the cluster                                                             | `bool`     | `false`      |
+| `externalMethod`          | Specify method to pass through the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` | `string`   | `{}`         |
+| `externalPorts`           | Specify ports to forward from outside the cluster                                                           | `[]int`    | `[22]`       |
+| `running`                 | if the virtual machine should be running                                                                    | `bool`     | `true`       |
+| `instanceType`            | Virtual Machine instance type                                                                               | `string`   | `u1.medium`  |
+| `instanceProfile`         | Virtual Machine preferences profile                                                                         | `string`   | `ubuntu`     |
+| `systemDisk`              | System disk configuration                                                                                   | `object`   | `{}`         |
+| `systemDisk.image`        | The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos`  | `string`   | `ubuntu`     |
+| `systemDisk.storage`      | The size of the disk allocated for the virtual machine                                                      | `string`   | `5Gi`        |
+| `systemDisk.storageClass` | StorageClass used to store the data                                                                         | `*string`  | `replicated` |
+| `gpus`                    | List of GPUs to attach                                                                                      | `[]object` | `[]`         |
+| `gpus[i].name`            | The name of the GPU to attach. This should match the GPU resource name in the cluster.                      | `string`   | `""`         |
+| `resources`               | Resources                                                                                                   | `object`   | `{}`         |
+| `resources.cpu`           | The number of CPU cores allocated to the virtual machine                                                    | `*string`  | `null`       |
+| `resources.sockets`       | The number of CPU sockets allocated to the virtual machine (used to define vCPU topology)                   | `*string`  | `null`       |
+| `resources.memory`        | The amount of memory allocated to the virtual machine                                                       | `*string`  | `null`       |
+| `sshKeys`                 | List of SSH public keys for authentication. Can be a single key or a list of keys.                          | `[]string` | `[]`         |
+| `cloudInit`               | cloud-init user data config. See cloud-init documentation for more details.                                 | `string`   | `""`         |
+| `cloudInitSeed`           | A seed string to generate an SMBIOS UUID for the VM.                                                        | `string`   | `""`         |
+
 
 ## U Series
 

packages/apps/virtual-machine/values.schema.json

@@ -1,50 +1,60 @@
 {
+  "title": "Chart Values",
+  "type": "object",
   "properties": {
     "cloudInit": {
-      "default": "",
       "description": "cloud-init user data config. See cloud-init documentation for more details.",
       "type": "string"
     },
     "cloudInitSeed": {
-      "default": "",
       "description": "A seed string to generate an SMBIOS UUID for the VM.",
       "type": "string"
     },
     "external": {
-      "default": false,
       "description": "Enable external access from outside the cluster",
-      "type": "boolean"
+      "type": "boolean",
+      "default": false
     },
     "externalMethod": {
-      "default": "PortList",
-      "description": "specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList`",
+      "description": "Specify method to pass through the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList`",
       "type": "string",
+      "default": "PortList",
       "enum": [
         "PortList",
         "WholeIP"
       ]
     },
     "externalPorts": {
-      "default": [],
       "description": "Specify ports to forward from outside the cluster",
+      "type": "array",
+      "default": [
+        22
+      ],
       "items": {
         "type": "integer"
-      },
-      "type": "array"
+      }
     },
     "gpus": {
-      "default": [],
       "description": "List of GPUs to attach",
+      "type": "array",
+      "default": [],
       "items": {
-        "type": "object"
-      },
-      "type": "array"
+        "type": "object",
+        "required": [
+          "name"
+        ],
+        "properties": {
+          "name": {
+            "description": "The name of the GPU to attach. This should match the GPU resource name in the cluster.",
+            "type": "string"
+          }
+        }
+      }
     },
     "instanceProfile": {
-      "default": "ubuntu",
       "description": "Virtual Machine preferences profile",
       "type": "string",
-      "optional": true,
+      "default": "ubuntu",
       "enum": [
         "alpine",
         "centos.7",
@@ -92,99 +102,65 @@
       ]
     },
     "instanceType": {
-      "default": "u1.medium",
       "description": "Virtual Machine instance type",
       "type": "string",
-      "optional": true,
-      "enum": [
-        "cx1.2xlarge",
-        "cx1.4xlarge",
-        "cx1.8xlarge",
-        "cx1.large",
-        "cx1.medium",
-        "cx1.xlarge",
-        "gn1.2xlarge",
-        "gn1.4xlarge",
-        "gn1.8xlarge",
-        "gn1.xlarge",
-        "m1.2xlarge",
-        "m1.4xlarge",
-        "m1.8xlarge",
-        "m1.large",
-        "m1.xlarge",
-        "n1.2xlarge",
-        "n1.4xlarge",
-        "n1.8xlarge",
-        "n1.large",
-        "n1.medium",
-        "n1.xlarge",
-        "o1.2xlarge",
-        "o1.4xlarge",
-        "o1.8xlarge",
-        "o1.large",
-        "o1.medium",
-        "o1.micro",
-        "o1.nano",
-        "o1.small",
-        "o1.xlarge",
-        "rt1.2xlarge",
-        "rt1.4xlarge",
-        "rt1.8xlarge",
-        "rt1.large",
-        "rt1.medium",
-        "rt1.micro",
-        "rt1.small",
-        "rt1.xlarge",
-        "u1.2xlarge",
-        "u1.2xmedium",
-        "u1.4xlarge",
-        "u1.8xlarge",
-        "u1.large",
-        "u1.medium",
-        "u1.micro",
-        "u1.nano",
-        "u1.small",
-        "u1.xlarge",
-        ""
-      ]
+      "default": "u1.medium"
     },
     "resources": {
+      "description": "Resources",
+      "type": "object",
+      "default": {},
       "properties": {
         "cpu": {
-          "default": "",
           "description": "The number of CPU cores allocated to the virtual machine",
-          "type": "string"
+          "type": "string",
+          "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+          "x-kubernetes-int-or-string": true
         },
         "memory": {
-          "default": "",
           "description": "The amount of memory allocated to the virtual machine",
-          "type": "string"
+          "type": "string",
+          "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+          "x-kubernetes-int-or-string": true
         },
         "sockets": {
-          "default": "",
           "description": "The number of CPU sockets allocated to the virtual machine (used to define vCPU topology)",
-          "type": "string"
+          "type": "string",
+          "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+          "x-kubernetes-int-or-string": true
         }
-      },
-      "type": "object"
+      }
     },
     "running": {
-      "default": true,
-      "description": "Determines if the virtual machine should be running",
-      "type": "boolean"
+      "description": "if the virtual machine should be running",
+      "type": "boolean",
+      "default": true
     },
     "sshKeys": {
-      "default": [],
       "description": "List of SSH public keys for authentication. Can be a single key or a list of keys.",
-      "items": {},
-      "type": "array"
+      "type": "array",
+      "default": [],
+      "items": {
+        "type": "string"
+      }
     },
     "systemDisk": {
+      "description": "System disk configuration",
+      "type": "object",
+      "default": {
+        "image": "ubuntu",
+        "storage": "5Gi",
+        "storageClass": "replicated"
+      },
+      "required": [
+        "image",
+        "storage"
+      ],
       "properties": {
         "image": {
-          "default": "ubuntu",
           "description": "The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos`",
           "type": "string",
+          "default": "ubuntu",
           "enum": [
             "ubuntu",
             "cirros",
@@ -194,19 +170,16 @@
           ]
         },
         "storage": {
-          "default": "5Gi",
           "description": "The size of the disk allocated for the virtual machine",
-          "type": "string"
+          "type": "string",
+          "default": "5Gi"
         },
         "storageClass": {
-          "default": "replicated",
           "description": "StorageClass used to store the data",
-          "type": "string"
+          "type": "string",
+          "default": "replicated"
         }
-      },
-      "type": "object"
+      }
     }
-  },
-  "title": "Chart Values",
-  "type": "object"
+  }
 }

packages/apps/virtual-machine/values.yaml

@@ -1,46 +1,53 @@
+
 ## @section Common parameters
-
-## @param external Enable external access from outside the cluster
-## @param externalMethod specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList`
-## @param externalPorts [array] Specify ports to forward from outside the cluster
+##
+## @param external {bool} Enable external access from outside the cluster
 external: false
-externalMethod: PortList
+## @param externalMethod {string enum:"PortList,WholeIP"} Specify method to pass through the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList`
+externalMethod: "PortList"
+## @param externalPorts {[]int} Specify ports to forward from outside the cluster
 externalPorts:
-- 22
+  - 22
 
-## @param running Determines if the virtual machine should be running
+## @param running {bool} if the virtual machine should be running
 running: true
 
-## @param instanceType Virtual Machine instance type
-## @param instanceProfile Virtual Machine preferences profile
+## @param instanceType {string} Virtual Machine instance type
+## @param instanceProfile {string} Virtual Machine preferences profile
 ##
 instanceType: "u1.medium"
 instanceProfile: ubuntu
 
-## @param systemDisk.image The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos`
-## @param systemDisk.storage The size of the disk allocated for the virtual machine
-## @param systemDisk.storageClass StorageClass used to store the data
-## 
+## @param systemDisk {systemDisk} System disk configuration
+## @field systemDisk.image {string enum:"ubuntu,cirros,alpine,fedora,talos"} The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos`
+## @field systemDisk.storage {string} The size of the disk allocated for the virtual machine
+## @field systemDisk.storageClass {*string} StorageClass used to store the data
+##
 systemDisk:
   image: ubuntu
   storage: 5Gi
   storageClass: replicated
 
-## @param gpus [array] List of GPUs to attach
+## @param gpus {[]gpu} List of GPUs to attach
+## @field gpu.name {string} The name of the GPU to attach. This should match the GPU resource name in the cluster.
 ## Example:
 ## gpus:
 ## - name: nvidia.com/GA102GL_A10
 gpus: []
 
-## @param resources.cpu The number of CPU cores allocated to the virtual machine
-## @param resources.memory The amount of memory allocated to the virtual machine
-## @param resources.sockets The number of CPU sockets allocated to the virtual machine (used to define vCPU topology)
-resources:
-  cpu: ""
-  memory: ""
-  sockets: ""
 
-## @param sshKeys [array] List of SSH public keys for authentication. Can be a single key or a list of keys.
+## @param resources {resources} Resources
+## @field resources.cpu {*quantity} The number of CPU cores allocated to the virtual machine
+## @field resources.sockets {*quantity} The number of CPU sockets allocated to the virtual machine (used to define vCPU topology)
+## @field resources.memory {*quantity} The amount of memory allocated to the virtual machine
+## Example:
+## resources:
+##  cpu: "4"
+##  sockets: "1"
+##  memory: "8Gi"
+resources: {}
+
+## @param sshKeys {[]string} List of SSH public keys for authentication. Can be a single key or a list of keys.
 ## Example:
 ## sshKeys:
 ##   - ssh-rsa ...
@@ -48,7 +55,7 @@ resources:
 ##
 sshKeys: []
 
-## @param cloudInit cloud-init user data config. See cloud-init documentation for more details.
+## @param cloudInit {string} cloud-init user data config. See cloud-init documentation for more details.
 ## - https://cloudinit.readthedocs.io/en/latest/explanation/format.html
 ## - https://cloudinit.readthedocs.io/en/latest/reference/examples.html
 ## Example:
@@ -59,11 +66,11 @@ sshKeys: []
 ##
 cloudInit: ""
 
-## @param cloudInitSeed A seed string to generate an SMBIOS UUID for the VM.
-cloudInitSeed: ""
+## @param cloudInitSeed {string} A seed string to generate an SMBIOS UUID for the VM.
 ## Change it to any new value to force a full cloud-init reconfiguration. Change it when you want to apply
 ## to an existing VM settings that are usually written only once, like new SSH keys or new network configuration.
 ## An empty value does nothing (and the existing UUID is not reverted). Please note that changing this value
 ## does not trigger a VM restart. You must perform the restart separately.
 ## Example:
 ## cloudInitSeed: "upd1"
+cloudInitSeed: ""

packages/apps/vm-instance/Chart.yaml

@@ -17,7 +17,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.10.1
+version: 0.10.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/vm-instance/Makefile

@@ -4,9 +4,9 @@ generate:
 	readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md
 	yq -o json -i '.properties.disks.items.type = "object" | .properties.disks.default = []' values.schema.json
 	yq -o json -i '.properties.gpus.items.type = "object" | .properties.gpus.default = []' values.schema.json
-	INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \
-	  && yq -i -o json ".properties.instanceType.optional=true | .properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json
+	#INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \
+	#  && yq -i -o json ".properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json
 	PREFERENCES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/preferences.yaml | yq 'split(" ") | . + [""]' -o json) \
-	  && yq -i -o json ".properties.instanceProfile.optional=true | .properties.instanceProfile.enum = $${PREFERENCES}" values.schema.json
+	  && yq -i -o json ".properties.instanceProfile.enum = $${PREFERENCES}" values.schema.json
 	yq -i -o json '.properties.externalPorts.items.type = "integer"' values.schema.json
 	yq -i -o json '.properties.externalMethod.enum = ["PortList", "WholeIP"]' values.schema.json

packages/apps/vm-instance/values.schema.json

@@ -52,7 +52,6 @@
       "default": "ubuntu",
       "description": "Virtual Machine preferences profile",
       "type": "string",
-      "optional": true,
       "enum": [
         "alpine",
         "centos.7",
@@ -102,59 +101,7 @@
     "instanceType": {
       "default": "u1.medium",
       "description": "Virtual Machine instance type",
-      "type": "string",
-      "optional": true,
-      "enum": [
-        "cx1.2xlarge",
-        "cx1.4xlarge",
-        "cx1.8xlarge",
-        "cx1.large",
-        "cx1.medium",
-        "cx1.xlarge",
-        "gn1.2xlarge",
-        "gn1.4xlarge",
-        "gn1.8xlarge",
-        "gn1.xlarge",
-        "m1.2xlarge",
-        "m1.4xlarge",
-        "m1.8xlarge",
-        "m1.large",
-        "m1.xlarge",
-        "n1.2xlarge",
-        "n1.4xlarge",
-        "n1.8xlarge",
-        "n1.large",
-        "n1.medium",
-        "n1.xlarge",
-        "o1.2xlarge",
-        "o1.4xlarge",
-        "o1.8xlarge",
-        "o1.large",
-        "o1.medium",
-        "o1.micro",
-        "o1.nano",
-        "o1.small",
-        "o1.xlarge",
-        "rt1.2xlarge",
-        "rt1.4xlarge",
-        "rt1.8xlarge",
-        "rt1.large",
-        "rt1.medium",
-        "rt1.micro",
-        "rt1.small",
-        "rt1.xlarge",
-        "u1.2xlarge",
-        "u1.2xmedium",
-        "u1.4xlarge",
-        "u1.8xlarge",
-        "u1.large",
-        "u1.medium",
-        "u1.micro",
-        "u1.nano",
-        "u1.small",
-        "u1.xlarge",
-        ""
-      ]
+      "type": "string"
     },
     "resources": {
       "properties": {

packages/core/installer/values.yaml

@@ -1,2 +1,2 @@
 cozystack:
-  image: ghcr.io/cozystack/cozystack/installer:v0.35.0-alpha.1@sha256:c50451e26a1a2a9f8962e26f6ab668b71b95186f53c1ae84118733e2cd464293
+  image: ghcr.io/cozystack/cozystack/installer:v0.35.0-alpha.3@sha256:b43615350697dbbc377d31152b290722c2bba7a053fd7c69403d666e6c7d53ee

packages/core/testing/images/e2e-sandbox/Dockerfile

@@ -19,6 +19,7 @@ RUN curl -sSL "https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_${TA
  && chmod +x /usr/local/bin/yq
 RUN curl -sSL "https://fluxcd.io/install.sh" | bash
 RUN curl -sSL "https://github.com/cozystack/cozypkg/raw/refs/heads/main/hack/install.sh" | sh -s -- -v "${COZYPKG_VERSION}"
-
+RUN curl https://dl.min.io/client/mc/release/${TARGETOS}-${TARGETARCH}/mc --create-dirs -o  /usr/local/bin/mc \
+&& chmod +x /usr/local/bin/mc
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

packages/core/testing/values.yaml

@@ -1,2 +1,2 @@
 e2e:
-  image: ghcr.io/cozystack/cozystack/e2e-sandbox:v0.35.0-alpha.1@sha256:ea73a2aeeac810be8eaee636aba6d6d10051772297ee76b6ce28a2aa7aa6a7f4
+  image: ghcr.io/cozystack/cozystack/e2e-sandbox:v0.35.0-alpha.3@sha256:cd6d65230150171c0e3934f71fcc270718f2bfec7509989b9d4d996ddef916d9

packages/extra/bootbox/images/matchbox.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/matchbox:v0.35.0-alpha.1@sha256:103e7c66ba47caee2126eca2d4b40e67041d4a3e6b729945b4dfc55decda74e7
+ghcr.io/cozystack/cozystack/matchbox:v0.35.0-alpha.3@sha256:043051d7c368b73943051b51bacfb3e383541b53190f74452d552ea2f30e420e

packages/extra/monitoring/Chart.yaml

@@ -3,4 +3,4 @@ name: monitoring
 description: Monitoring and observability stack
 icon: /logos/monitoring.svg
 type: application
-version: 1.12.0
+version: 1.12.1

packages/extra/monitoring/Makefile

@@ -6,11 +6,7 @@ include ../../../scripts/common-envs.mk
 include ../../../scripts/package.mk
 
 generate:
-	readme-generator-for-helm -v values.yaml -s values.schema.json.tmp -r README.md
-	cat values.schema.json.tmp | \
-		jq '.properties.metricsStorages.items.type = "object" | .properties.logsStorages.items.type = "object"' \
-		> values.schema.json
-	rm -f values.schema.json.tmp
+	cozyvalues-gen -v values.yaml -s values.schema.json -r README.md
 
 image:
 	docker buildx build images/grafana \

packages/extra/monitoring/README.md

@@ -4,22 +4,88 @@
 
 ### Common parameters
 
-| Name                                      | Description                                                                                               | Value   |
-| ----------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------- |
-| `host`                                    | The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host). | `""`    |
-| `metricsStorages`                         | Configuration of metrics storage instances                                                                | `[]`    |
-| `logsStorages`                            | Configuration of logs storage instances                                                                   | `[]`    |
-| `alerta.storage`                          | Persistent Volume size for alerta database                                                                | `10Gi`  |
-| `alerta.storageClassName`                 | StorageClass used to store the data                                                                       | `""`    |
-| `alerta.resources.requests.cpu`           | The minimum amount of CPU required for alerta                                                             | `100m`  |
-| `alerta.resources.requests.memory`        | The minimum amount of memory required for alerta                                                          | `256Mi` |
-| `alerta.resources.limits.cpu`             | The maximum amount of CPU allowed for alerta                                                              | `1`     |
-| `alerta.resources.limits.memory`          | The maximum amount of memory allowed for alerta                                                           | `1Gi`   |
-| `alerta.alerts.telegram.token`            | telegram token for your bot                                                                               | `""`    |
-| `alerta.alerts.telegram.chatID`           | specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot                       | `""`    |
-| `alerta.alerts.telegram.disabledSeverity` | list of severity without alerts, separated comma like: "informational,warning"                            | `""`    |
-| `grafana.db.size`                         | Persistent Volume size for grafana database                                                               | `10Gi`  |
-| `grafana.resources.requests.cpu`          | The minimum amount of CPU required for grafana                                                            | `100m`  |
-| `grafana.resources.requests.memory`       | The minimum amount of memory required for grafana                                                         | `256Mi` |
-| `grafana.resources.limits.cpu`            | The maximum amount of CPU allowed for grafana                                                             | `1`     |
-| `grafana.resources.limits.memory`         | The maximum amount of memory allowed for grafana                                                          | `1Gi`   |
+| Name   | Description                                                                                               | Type     | Value |
+| ------ | --------------------------------------------------------------------------------------------------------- | -------- | ----- |
+| `host` | The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host). | `string` | `""`  |
+
+
+### Metrics storage configuration
+
+| Name                                             | Description                                                    | Type       | Value   |
+| ------------------------------------------------ | -------------------------------------------------------------- | ---------- | ------- |
+| `metricsStorages`                                | Configuration of metrics storage instances                     | `[]object` | `[...]` |
+| `metricsStorages[i].name`                        | Name of the storage instance                                   | `string`   | `""`    |
+| `metricsStorages[i].retentionPeriod`             | Retention period for the metrics in the storage instance       | `string`   | `""`    |
+| `metricsStorages[i].deduplicationInterval`       | Deduplication interval for the metrics in the storage instance | `string`   | `""`    |
+| `metricsStorages[i].storage`                     | Persistent Volume size for the storage instance                | `string`   | `""`    |
+| `metricsStorages[i].storageClassName`            | StorageClass used to store the data                            | `*string`  | `null`  |
+| `metricsStorages[i].vminsert`                    | Configuration for vminsert component of the storage instance   | `*object`  | `null`  |
+| `metricsStorages[i].vminsert.minAllowed`         | Requests (minimum allowed/available resources)                 | `*object`  | `null`  |
+| `metricsStorages[i].vminsert.minAllowed.cpu`     | CPU request (minimum available CPU)                            | `*string`  | `null`  |
+| `metricsStorages[i].vminsert.minAllowed.memory`  | Memory request (minimum available memory)                      | `*string`  | `null`  |
+| `metricsStorages[i].vminsert.maxAllowed`         | Limits (maximum allowed/available resources )                  | `*object`  | `null`  |
+| `metricsStorages[i].vminsert.maxAllowed.cpu`     | CPU limit (maximum available CPU)                              | `*string`  | `null`  |
+| `metricsStorages[i].vminsert.maxAllowed.memory`  | Memory limit (maximum available memory)                        | `*string`  | `null`  |
+| `metricsStorages[i].vmselect`                    | Configuration for vmselect component of the storage instance   | `*object`  | `null`  |
+| `metricsStorages[i].vmselect.minAllowed`         | Requests (minimum allowed/available resources)                 | `*object`  | `null`  |
+| `metricsStorages[i].vmselect.minAllowed.cpu`     | CPU request (minimum available CPU)                            | `*string`  | `null`  |
+| `metricsStorages[i].vmselect.minAllowed.memory`  | Memory request (minimum available memory)                      | `*string`  | `null`  |
+| `metricsStorages[i].vmselect.maxAllowed`         | Limits (maximum allowed/available resources )                  | `*object`  | `null`  |
+| `metricsStorages[i].vmselect.maxAllowed.cpu`     | CPU limit (maximum available CPU)                              | `*string`  | `null`  |
+| `metricsStorages[i].vmselect.maxAllowed.memory`  | Memory limit (maximum available memory)                        | `*string`  | `null`  |
+| `metricsStorages[i].vmstorage`                   | Configuration for vmstorage component of the storage instance  | `*object`  | `null`  |
+| `metricsStorages[i].vmstorage.minAllowed`        | Requests (minimum allowed/available resources)                 | `*object`  | `null`  |
+| `metricsStorages[i].vmstorage.minAllowed.cpu`    | CPU request (minimum available CPU)                            | `*string`  | `null`  |
+| `metricsStorages[i].vmstorage.minAllowed.memory` | Memory request (minimum available memory)                      | `*string`  | `null`  |
+| `metricsStorages[i].vmstorage.maxAllowed`        | Limits (maximum allowed/available resources )                  | `*object`  | `null`  |
+| `metricsStorages[i].vmstorage.maxAllowed.cpu`    | CPU limit (maximum available CPU)                              | `*string`  | `null`  |
+| `metricsStorages[i].vmstorage.maxAllowed.memory` | Memory limit (maximum available memory)                        | `*string`  | `null`  |
+
+
+### Logs storage configuration
+
+| Name                               | Description                                           | Type       | Value   |
+| ---------------------------------- | ----------------------------------------------------- | ---------- | ------- |
+| `logsStorages`                     | Configuration of logs storage instances               | `[]object` | `[...]` |
+| `logsStorages[i].name`             | Name of the storage instance                          | `string`   | `""`    |
+| `logsStorages[i].retentionPeriod`  | Retention period for the logs in the storage instance | `string`   | `""`    |
+| `logsStorages[i].storage`          | Persistent Volume size for the storage instance       | `string`   | `""`    |
+| `logsStorages[i].storageClassName` | StorageClass used to store the data                   | `*string`  | `null`  |
+
+
+### Alerta configuration
+
+| Name                                      | Description                                                                         | Type      | Value   |
+| ----------------------------------------- | ----------------------------------------------------------------------------------- | --------- | ------- |
+| `alerta`                                  | Configuration for Alerta service                                                    | `object`  | `{}`    |
+| `alerta.storage`                          | Persistent Volume size for the database                                             | `string`  | `10Gi`  |
+| `alerta.storageClassName`                 | StorageClass used to store the data                                                 | `string`  | `""`    |
+| `alerta.resources`                        | Resources configuration                                                             | `*object` | `null`  |
+| `alerta.resources.requests`               |                                                                                     | `*object` | `null`  |
+| `alerta.resources.requests.cpu`           | CPU request (minimum available CPU)                                                 | `*string` | `100m`  |
+| `alerta.resources.requests.memory`        | Memory request (minimum available memory)                                           | `*string` | `256Mi` |
+| `alerta.resources.limits`                 |                                                                                     | `*object` | `null`  |
+| `alerta.resources.limits.cpu`             | CPU limit (maximum available CPU)                                                   | `*string` | `1`     |
+| `alerta.resources.limits.memory`          | Memory limit (maximum available memory)                                             | `*string` | `1Gi`   |
+| `alerta.alerts`                           | Configuration for alerts                                                            | `object`  | `{}`    |
+| `alerta.alerts.telegram`                  | Configuration for Telegram alerts                                                   | `object`  | `{}`    |
+| `alerta.alerts.telegram.token`            | Telegram token for your bot                                                         | `string`  | `""`    |
+| `alerta.alerts.telegram.chatID`           | Specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot | `string`  | `""`    |
+| `alerta.alerts.telegram.disabledSeverity` | List of severity without alerts, separated by comma like: "informational,warning"   | `string`  | `""`    |
+
+
+### Grafana configuration
+
+| Name                                | Description                               | Type      | Value   |
+| ----------------------------------- | ----------------------------------------- | --------- | ------- |
+| `grafana`                           | Configuration for Grafana                 | `object`  | `{}`    |
+| `grafana.db`                        | Database configuration                    | `object`  | `{}`    |
+| `grafana.db.size`                   | Persistent Volume size for the database   | `string`  | `10Gi`  |
+| `grafana.resources`                 | Resources configuration                   | `*object` | `null`  |
+| `grafana.resources.requests`        |                                           | `*object` | `null`  |
+| `grafana.resources.requests.cpu`    | CPU request (minimum available CPU)       | `*string` | `100m`  |
+| `grafana.resources.requests.memory` | Memory request (minimum available memory) | `*string` | `256Mi` |
+| `grafana.resources.limits`          |                                           | `*object` | `null`  |
+| `grafana.resources.limits.cpu`      | CPU limit (maximum available CPU)         | `*string` | `1`     |
+| `grafana.resources.limits.memory`   | Memory limit (maximum available memory)   | `*string` | `1Gi`   |
+

packages/extra/monitoring/dashboards.list

@@ -37,4 +37,5 @@ flux/flux-stats
 kafka/strimzi-kafka
 goldpinger/goldpinger
 clickhouse/altinity-clickhouse-operator-dashboard
-storage/linstor
+storage/linstor
+seaweedfs/seaweedfs

packages/extra/monitoring/images/grafana.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/grafana:1.12.0@sha256:c63978e1ed0304e8518b31ddee56c4e8115541b997d8efbe1c0a74da57140399
+ghcr.io/cozystack/cozystack/grafana:1.12.1@sha256:c63978e1ed0304e8518b31ddee56c4e8115541b997d8efbe1c0a74da57140399

packages/extra/monitoring/values.schema.json

@@ -1,152 +1,487 @@
 {
+  "title": "Chart Values",
+  "type": "object",
   "properties": {
     "alerta": {
+      "description": "Configuration for Alerta service",
+      "type": "object",
+      "default": {
+        "alerts": {
+          "telegram": {
+            "chatID": "",
+            "disabledSeverity": "",
+            "token": ""
+          }
+        },
+        "resources": {
+          "limits": {
+            "cpu": "1",
+            "memory": "1Gi"
+          },
+          "requests": {
+            "cpu": "100m",
+            "memory": "256Mi"
+          }
+        },
+        "storage": "10Gi",
+        "storageClassName": ""
+      },
+      "required": [
+        "alerts",
+        "storage",
+        "storageClassName"
+      ],
       "properties": {
         "alerts": {
+          "description": "Configuration for alerts",
+          "type": "object",
+          "default": {
+            "telegram": {
+              "chatID": "",
+              "disabledSeverity": "",
+              "token": ""
+            }
+          },
+          "required": [
+            "telegram"
+          ],
           "properties": {
             "telegram": {
+              "description": "Configuration for Telegram alerts",
+              "type": "object",
+              "default": {
+                "chatID": "",
+                "disabledSeverity": "",
+                "token": ""
+              },
+              "required": [
+                "chatID",
+                "disabledSeverity",
+                "token"
+              ],
               "properties": {
                 "chatID": {
-                  "default": "",
-                  "description": "specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot",
+                  "description": "Specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot",
                   "type": "string"
                 },
                 "disabledSeverity": {
-                  "default": "",
-                  "description": "list of severity without alerts, separated comma like: \"informational,warning\"",
+                  "description": "List of severity without alerts, separated by comma like: \"informational,warning\"",
                   "type": "string"
                 },
                 "token": {
-                  "default": "",
-                  "description": "telegram token for your bot",
+                  "description": "Telegram token for your bot",
                   "type": "string"
                 }
-              },
-              "type": "object"
+              }
             }
-          },
-          "type": "object"
+          }
         },
         "resources": {
-          "properties": {
+          "description": "Resources configuration",
+          "type": "object",
+          "default": {
             "limits": {
-              "properties": {
-                "cpu": {
-                  "default": "1",
-                  "description": "The maximum amount of CPU allowed for alerta",
-                  "type": "string"
-                },
-                "memory": {
-                  "default": "1Gi",
-                  "description": "The maximum amount of memory allowed for alerta",
-                  "type": "string"
-                }
-              },
-              "type": "object"
+              "cpu": "1",
+              "memory": "1Gi"
             },
             "requests": {
-              "properties": {
-                "cpu": {
-                  "default": "100m",
-                  "description": "The minimum amount of CPU required for alerta",
-                  "type": "string"
-                },
-                "memory": {
-                  "default": "256Mi",
-                  "description": "The minimum amount of memory required for alerta",
-                  "type": "string"
-                }
-              },
-              "type": "object"
+              "cpu": "100m",
+              "memory": "256Mi"
             }
           },
-          "type": "object"
+          "properties": {
+            "limits": {
+              "type": "object",
+              "default": {
+                "cpu": "1",
+                "memory": "1Gi"
+              },
+              "properties": {
+                "cpu": {
+                  "description": "CPU limit (maximum available CPU)",
+                  "type": "string",
+                  "default": "1",
+                  "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                  "x-kubernetes-int-or-string": true
+                },
+                "memory": {
+                  "description": "Memory limit (maximum available memory)",
+                  "type": "string",
+                  "default": "1Gi",
+                  "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                  "x-kubernetes-int-or-string": true
+                }
+              }
+            },
+            "requests": {
+              "type": "object",
+              "default": {
+                "cpu": "100m",
+                "memory": "256Mi"
+              },
+              "properties": {
+                "cpu": {
+                  "description": "CPU request (minimum available CPU)",
+                  "type": "string",
+                  "default": "100m",
+                  "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                  "x-kubernetes-int-or-string": true
+                },
+                "memory": {
+                  "description": "Memory request (minimum available memory)",
+                  "type": "string",
+                  "default": "256Mi",
+                  "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                  "x-kubernetes-int-or-string": true
+                }
+              }
+            }
+          }
         },
         "storage": {
-          "default": "10Gi",
-          "description": "Persistent Volume size for alerta database",
-          "type": "string"
+          "description": "Persistent Volume size for the database",
+          "type": "string",
+          "default": "10Gi"
         },
         "storageClassName": {
-          "default": "",
           "description": "StorageClass used to store the data",
           "type": "string"
         }
-      },
-      "type": "object"
+      }
     },
     "grafana": {
-      "properties": {
+      "description": "Configuration for Grafana",
+      "type": "object",
+      "default": {
         "db": {
-          "properties": {
-            "size": {
-              "default": "10Gi",
-              "description": "Persistent Volume size for grafana database",
-              "type": "string"
-            }
-          },
-          "type": "object"
+          "size": "10Gi"
         },
         "resources": {
-          "properties": {
-            "limits": {
-              "properties": {
-                "cpu": {
-                  "default": "1",
-                  "description": "The maximum amount of CPU allowed for grafana",
-                  "type": "string"
-                },
-                "memory": {
-                  "default": "1Gi",
-                  "description": "The maximum amount of memory allowed for grafana",
-                  "type": "string"
-                }
-              },
-              "type": "object"
-            },
-            "requests": {
-              "properties": {
-                "cpu": {
-                  "default": "100m",
-                  "description": "The minimum amount of CPU required for grafana",
-                  "type": "string"
-                },
-                "memory": {
-                  "default": "256Mi",
-                  "description": "The minimum amount of memory required for grafana",
-                  "type": "string"
-                }
-              },
-              "type": "object"
-            }
+          "limits": {
+            "cpu": "1",
+            "memory": "1Gi"
           },
-          "type": "object"
+          "requests": {
+            "cpu": "100m",
+            "memory": "256Mi"
+          }
         }
       },
-      "type": "object"
+      "required": [
+        "db"
+      ],
+      "properties": {
+        "db": {
+          "description": "Database configuration",
+          "type": "object",
+          "default": {
+            "size": "10Gi"
+          },
+          "required": [
+            "size"
+          ],
+          "properties": {
+            "size": {
+              "description": "Persistent Volume size for the database",
+              "type": "string",
+              "default": "10Gi"
+            }
+          }
+        },
+        "resources": {
+          "description": "Resources configuration",
+          "type": "object",
+          "default": {
+            "limits": {
+              "cpu": "1",
+              "memory": "1Gi"
+            },
+            "requests": {
+              "cpu": "100m",
+              "memory": "256Mi"
+            }
+          },
+          "properties": {
+            "limits": {
+              "type": "object",
+              "default": {
+                "cpu": "1",
+                "memory": "1Gi"
+              },
+              "properties": {
+                "cpu": {
+                  "description": "CPU limit (maximum available CPU)",
+                  "type": "string",
+                  "default": "1",
+                  "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                  "x-kubernetes-int-or-string": true
+                },
+                "memory": {
+                  "description": "Memory limit (maximum available memory)",
+                  "type": "string",
+                  "default": "1Gi",
+                  "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                  "x-kubernetes-int-or-string": true
+                }
+              }
+            },
+            "requests": {
+              "type": "object",
+              "default": {
+                "cpu": "100m",
+                "memory": "256Mi"
+              },
+              "properties": {
+                "cpu": {
+                  "description": "CPU request (minimum available CPU)",
+                  "type": "string",
+                  "default": "100m",
+                  "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                  "x-kubernetes-int-or-string": true
+                },
+                "memory": {
+                  "description": "Memory request (minimum available memory)",
+                  "type": "string",
+                  "default": "256Mi",
+                  "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                  "x-kubernetes-int-or-string": true
+                }
+              }
+            }
+          }
+        }
+      }
     },
     "host": {
-      "default": "",
       "description": "The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host).",
       "type": "string"
     },
     "logsStorages": {
-      "default": [],
       "description": "Configuration of logs storage instances",
+      "type": "array",
+      "default": [
+        {
+          "name": "generic",
+          "retentionPeriod": "1",
+          "storage": "10Gi",
+          "storageClassName": "replicated"
+        }
+      ],
       "items": {
-        "type": "object"
-      },
-      "type": "array"
+        "type": "object",
+        "required": [
+          "name",
+          "retentionPeriod",
+          "storage"
+        ],
+        "properties": {
+          "name": {
+            "description": "Name of the storage instance",
+            "type": "string"
+          },
+          "retentionPeriod": {
+            "description": "Retention period for the logs in the storage instance",
+            "type": "string"
+          },
+          "storage": {
+            "description": "Persistent Volume size for the storage instance",
+            "type": "string"
+          },
+          "storageClassName": {
+            "description": "StorageClass used to store the data",
+            "type": "string"
+          }
+        }
+      }
     },
     "metricsStorages": {
-      "default": [],
       "description": "Configuration of metrics storage instances",
+      "type": "array",
+      "default": [
+        {
+          "deduplicationInterval": "15s",
+          "name": "shortterm",
+          "retentionPeriod": "3d",
+          "storage": "10Gi",
+          "storageClassName": ""
+        },
+        {
+          "deduplicationInterval": "5m",
+          "name": "longterm",
+          "retentionPeriod": "14d",
+          "storage": "10Gi",
+          "storageClassName": ""
+        }
+      ],
       "items": {
-        "type": "object"
-      },
-      "type": "array"
+        "type": "object",
+        "required": [
+          "deduplicationInterval",
+          "name",
+          "retentionPeriod",
+          "storage"
+        ],
+        "properties": {
+          "deduplicationInterval": {
+            "description": "Deduplication interval for the metrics in the storage instance",
+            "type": "string"
+          },
+          "name": {
+            "description": "Name of the storage instance",
+            "type": "string"
+          },
+          "retentionPeriod": {
+            "description": "Retention period for the metrics in the storage instance",
+            "type": "string"
+          },
+          "storage": {
+            "description": "Persistent Volume size for the storage instance",
+            "type": "string"
+          },
+          "storageClassName": {
+            "description": "StorageClass used to store the data",
+            "type": "string"
+          },
+          "vminsert": {
+            "description": "Configuration for vminsert component of the storage instance",
+            "type": "object",
+            "properties": {
+              "maxAllowed": {
+                "description": "Limits (maximum allowed/available resources )",
+                "type": "object",
+                "properties": {
+                  "cpu": {
+                    "description": "CPU limit (maximum available CPU)",
+                    "type": "string",
+                    "default": "1",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  },
+                  "memory": {
+                    "description": "Memory limit (maximum available memory)",
+                    "type": "string",
+                    "default": "1Gi",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  }
+                }
+              },
+              "minAllowed": {
+                "description": "Requests (minimum allowed/available resources)",
+                "type": "object",
+                "properties": {
+                  "cpu": {
+                    "description": "CPU request (minimum available CPU)",
+                    "type": "string",
+                    "default": "100m",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  },
+                  "memory": {
+                    "description": "Memory request (minimum available memory)",
+                    "type": "string",
+                    "default": "256Mi",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  }
+                }
+              }
+            }
+          },
+          "vmselect": {
+            "description": "Configuration for vmselect component of the storage instance",
+            "type": "object",
+            "properties": {
+              "maxAllowed": {
+                "description": "Limits (maximum allowed/available resources )",
+                "type": "object",
+                "properties": {
+                  "cpu": {
+                    "description": "CPU limit (maximum available CPU)",
+                    "type": "string",
+                    "default": "1",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  },
+                  "memory": {
+                    "description": "Memory limit (maximum available memory)",
+                    "type": "string",
+                    "default": "1Gi",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  }
+                }
+              },
+              "minAllowed": {
+                "description": "Requests (minimum allowed/available resources)",
+                "type": "object",
+                "properties": {
+                  "cpu": {
+                    "description": "CPU request (minimum available CPU)",
+                    "type": "string",
+                    "default": "100m",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  },
+                  "memory": {
+                    "description": "Memory request (minimum available memory)",
+                    "type": "string",
+                    "default": "256Mi",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  }
+                }
+              }
+            }
+          },
+          "vmstorage": {
+            "description": "Configuration for vmstorage component of the storage instance",
+            "type": "object",
+            "properties": {
+              "maxAllowed": {
+                "description": "Limits (maximum allowed/available resources )",
+                "type": "object",
+                "properties": {
+                  "cpu": {
+                    "description": "CPU limit (maximum available CPU)",
+                    "type": "string",
+                    "default": "1",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  },
+                  "memory": {
+                    "description": "Memory limit (maximum available memory)",
+                    "type": "string",
+                    "default": "1Gi",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  }
+                }
+              },
+              "minAllowed": {
+                "description": "Requests (minimum allowed/available resources)",
+                "type": "object",
+                "properties": {
+                  "cpu": {
+                    "description": "CPU request (minimum available CPU)",
+                    "type": "string",
+                    "default": "100m",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  },
+                  "memory": {
+                    "description": "Memory request (minimum available memory)",
+                    "type": "string",
+                    "default": "256Mi",
+                    "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+                    "x-kubernetes-int-or-string": true
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
     }
-  },
-  "title": "Chart Values",
-  "type": "object"
-}
+  }
+}

packages/extra/monitoring/values.yaml

@@ -1,10 +1,29 @@
 ## @section Common parameters
 
-## @param host The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host).
+## @param host {string} The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host).
 host: ""
 
-## @param metricsStorages [array] Configuration of metrics storage instances
-##
+## @section Metrics storage configuration
+
+## @param metricsStorages {[]metricsStorage} Configuration of metrics storage instances
+## @field metricsStorage.name {string} Name of the storage instance
+## @field metricsStorage.retentionPeriod {string} Retention period for the metrics in the storage instance
+## @field metricsStorage.deduplicationInterval {string} Deduplication interval for the metrics in the storage instance
+## @field metricsStorage.storage {string} Persistent Volume size for the storage instance
+## @field metricsStorage.storageClassName {*string} StorageClass used to store the data
+## @field metricsStorage.vminsert {*vmcomponent} Configuration for vminsert component of the storage instance
+## @field metricsStorage.vmselect {*vmcomponent} Configuration for vmselect component of the storage instance
+## @field metricsStorage.vmstorage {*vmcomponent} Configuration for vmstorage component of the storage instance
+
+## @field request.cpu {*quantity} CPU request (minimum available CPU)
+## @field request.memory {*quantity} Memory request (minimum available memory)
+## @field limit.cpu {*quantity} CPU limit (maximum available CPU)
+## @field limit.memory {*quantity} Memory limit (maximum available memory)
+## @field vmcomponent.minAllowed {*request} Requests (minimum allowed/available resources)
+## @field vmcomponent.maxAllowed {*limit} Limits (maximum allowed/available resources )
+## @field resources.requests {*request}
+## @field resources.limits {*limit}
+
 ## Example:
 ## metricsStorages:
 ## - name: shortterm
@@ -46,7 +65,13 @@ metricsStorages:
   storage: 10Gi
   storageClassName: ""
 
-## @param logsStorages [array] Configuration of logs storage instances
+## @section Logs storage configuration
+
+## @param logsStorages {[]logsStorage} Configuration of logs storage instances
+## @field logsStorage.name {string} Name of the storage instance
+## @field logsStorage.retentionPeriod {string} Retention period for the logs in the storage instance
+## @field logsStorage.storage {string} Persistent Volume size for the storage instance
+## @field logsStorage.storageClassName {*string} StorageClass used to store the data
 ##
 logsStorages:
 - name: generic
@@ -54,14 +79,17 @@ logsStorages:
   storage: 10Gi
   storageClassName: replicated
 
-## Configuration for Alerta
-## @param alerta.storage Persistent Volume size for alerta database
-## @param alerta.storageClassName StorageClass used to store the data
-## @param alerta.resources.requests.cpu The minimum amount of CPU required for alerta
-## @param alerta.resources.requests.memory  The minimum amount of memory required for alerta
-## @param alerta.resources.limits.cpu The maximum amount of CPU allowed for alerta
-## @param alerta.resources.limits.memory The maximum amount of memory allowed for alerta
-##
+## @section Alerta configuration
+
+## @param alerta {alerta} Configuration for Alerta service
+## @field alerta.storage {string} Persistent Volume size for the database
+## @field alerta.storageClassName {string} StorageClass used to store the data
+## @field alerta.resources {*resources} Resources configuration
+## @field alerta.alerts {alerts} Configuration for alerts
+## @field alerts.telegram {telegramAlerts} Configuration for Telegram alerts
+## @field telegramAlerts.token {string} Telegram token for your bot
+## @field telegramAlerts.chatID {string} Specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot
+## @field telegramAlerts.disabledSeverity {string} List of severity without alerts, separated by comma like: "informational,warning"
 alerta:
   storage: 10Gi
   storageClassName: ""
@@ -73,9 +101,6 @@ alerta:
       cpu: 100m
       memory: 256Mi
   alerts:
-    ## @param alerta.alerts.telegram.token telegram token for your bot
-    ## @param alerta.alerts.telegram.chatID specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot
-    ## @param alerta.alerts.telegram.disabledSeverity list of severity without alerts, separated comma like: "informational,warning"
     ## example:
     ##   telegram:
     ##     token: "7262461387:AAGtwq16iwuVtWtzoN6TUEMpF00fpC9Xz34"
@@ -87,12 +112,14 @@ alerta:
       chatID: ""
       disabledSeverity: ""
 
-## Configuration for Grafana
-## @param grafana.db.size Persistent Volume size for grafana database
-## @param grafana.resources.requests.cpu The minimum amount of CPU required for grafana
-## @param grafana.resources.requests.memory  The minimum amount of memory required for grafana
-## @param grafana.resources.limits.cpu The maximum amount of CPU allowed for grafana
-## @param grafana.resources.limits.memory The maximum amount of memory allowed for grafana
+## @section Grafana configuration
+
+## @param grafana {grafana} Configuration for Grafana
+## @field grafana.db {grafanaDB} Database configuration
+## @field grafanaDB.size {string} Persistent Volume size for the database
+## @field grafana.resources {*resources} Resources configuration
+
+
 grafana:
   db:
     size: 10Gi

packages/extra/seaweedfs/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.0
+version: 0.6.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/extra/seaweedfs/README.md

@@ -4,14 +4,15 @@
 
 ### Common parameters
 
-| Name                | Description                                                                                                              | Value           |
-| ------------------- | ------------------------------------------------------------------------------------------------------------------------ | --------------- |
-| `host`              | The hostname used to access the SeaweedFS externally (defaults to 's3' subdomain for the tenant host).                   | `""`            |
-| `topology`          | The topology of the SeaweedFS cluster. (allowed values: Simple, MultiZone, Client)                                       | `Simple`        |
-| `replicationFactor` | The number of replicas for each volume in the SeaweedFS cluster.                                                         | `2`             |
-| `replicas`          | Persistent Volume size for SeaweedFS                                                                                     | `2`             |
-| `size`              | Persistent Volume size                                                                                                   | `10Gi`          |
-| `storageClass`      | StorageClass used to store the data                                                                                      | `""`            |
-| `zones`             | A map of zones for MultiZone topology. Each zone can have its own number of replicas and size.                           | `{}`            |
-| `filer.external`    | Enable external access to the SeaweedFS filer from outside the cluster. Use this when `topology` is not set to `Client`. | `false`         |
-| `remoteEndpoint`    | The endpoint of the remote filer GRPC service. Used when `topology` is set to `Client`.                                  | `1.2.3.4:18888` |
+| Name                | Description                                                                                            | Value    |
+| ------------------- | ------------------------------------------------------------------------------------------------------ | -------- |
+| `host`              | The hostname used to access the SeaweedFS externally (defaults to 's3' subdomain for the tenant host). | `""`     |
+| `topology`          | The topology of the SeaweedFS cluster. (allowed values: Simple, MultiZone, Client)                     | `Simple` |
+| `replicationFactor` | The number of replicas for each volume in the SeaweedFS cluster.                                       | `2`      |
+| `replicas`          | Persistent Volume size for SeaweedFS                                                                   | `2`      |
+| `size`              | Persistent Volume size                                                                                 | `10Gi`   |
+| `storageClass`      | StorageClass used to store the data                                                                    | `""`     |
+| `zones`             | A map of zones for MultiZone topology. Each zone can have its own number of replicas and size.         | `{}`     |
+| `filer.grpcHost`    | The hostname used to expose or access the filer service externally.                                    | `""`     |
+| `filer.grpcPort`    | The port used to access the filer service externally.                                                  | `443`    |
+| `filer.whitelist`   | A list of IP addresses or CIDR ranges that are allowed to access the filer service.                    | `[]`     |

packages/extra/seaweedfs/images/objectstorage-sidecar.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/objectstorage-sidecar:v0.35.0-alpha.1@sha256:e4f9a7302285ea9febeb28fc2fa97cb7c01bb91e602f975c31aad1fe46f778f7
+ghcr.io/cozystack/cozystack/objectstorage-sidecar:v0.35.0-alpha.3@sha256:4a67ec40c13bf912151957dc0e7409f7e01a2859905285c98c20baae7b667ae2

packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag

@@ -0,0 +1 @@
+ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.2.0

packages/extra/seaweedfs/templates/client/cosi-deployment.yaml

@@ -35,7 +35,7 @@ spec:
     spec:
       containers:
       - name: seaweedfs-cosi-driver
-        image: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.1.2
+        image: "{{ $.Files.Get "images/seaweedfs-cosi-driver.tag" | trim }}"
         imagePullPolicy: IfNotPresent
         env:
         - name: DRIVERNAME
@@ -43,7 +43,7 @@ spec:
         - name: ENDPOINT
           value: https://{{ .Values.host | default (printf "s3.%s" $host) }}
         - name: SEAWEEDFS_FILER
-          value: {{ .Values.remoteEndpoint }}
+          value: "{{ .Values.filer.grpcHost }}:{{ .Values.filer.grpcPort }}"
         - name: WEED_GRPC_CLIENT_KEY
           value: /usr/local/share/ca-certificates/client/tls.key
         - name: WEED_GRPC_CLIENT_CERT

packages/extra/seaweedfs/templates/ingress.yaml

@@ -0,0 +1,44 @@
+{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
+{{- $issuerType := (index $cozyConfig.data "clusterissuer") | default "http01" }}
+
+{{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }}
+{{- $ingress := index $myNS.metadata.annotations "namespace.cozystack.io/ingress" }}
+{{- $host := index $myNS.metadata.annotations "namespace.cozystack.io/host" }}
+{{- if and (not (eq .Values.topology "Client")) (.Values.filer.grpcHost) }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/backend-protocol: GRPCS
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    nginx.ingress.kubernetes.io/whitelist-source-range: "{{ join "," (.Values.filer.whitelist | default "0.0.0.0/32") }}"
+  name: seaweedfs-filer-external
+spec:
+  ingressClassName: tenant-root
+  rules:
+  - host: {{ .Values.filer.grpcHost | default (printf "filer.%s" $host) }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: {{ $.Release.Name }}-filer-external
+            port:
+              number: 18888
+        path: /
+        pathType: ImplementationSpecific
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $.Release.Name }}-filer-external
+spec:
+  ports:
+  - name: swfs-filer-grpc
+    port: 18888
+    protocol: TCP
+    targetPort: 18888
+  selector:
+    app.kubernetes.io/component: filer
+    app.kubernetes.io/name: {{ $.Release.Name }}
+{{- end }}

packages/extra/seaweedfs/templates/seaweedfs.yaml

@@ -2,8 +2,8 @@
 {{- if not (has .Values.topology (list "Simple" "MultiZone" "Client")) }}
 {{-   fail "Invalid value for .Values.topology. Must be one of 'Simple', 'MultiZone' or 'Client'." }}
 {{- end }}
-{{- if and (eq .Values.topology "Client") (not .Values.remoteEndpoint) }}
-{{-   fail "When topology is 'Client', .Values.remoteEndpoint must be set to a valid remote filer GRPC service endpoint." }}
+{{- if and (eq .Values.topology "Client") (not .Values.filer.grpcHost) }}
+{{-   fail "When topology is 'Client', .Values.filer.grpcHost must be set to a valid remote filer GRPC service endpoint." }}
 {{- end }}
 {{- if lt (int .Values.replicationFactor) 1 }}
 {{-   fail "Invalid value for .Values.replicationFactor. Must be at least 1." }}

packages/extra/seaweedfs/templates/svc-external.yaml

@@ -1,18 +0,0 @@
-{{- if and (not (eq .Values.topology "Client")) (.Values.filer.external) }}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ $.Release.Name }}-filer-external
-spec:
-  type: LoadBalancer
-  externalTrafficPolicy: Local
-  ports:
-  - name: swfs-filer-grpc
-    port: 18888
-    protocol: TCP
-    targetPort: 18888
-  selector:
-    app.kubernetes.io/component: filer
-    app.kubernetes.io/name: {{ $.Release.Name }}
-{{- end }}

packages/extra/seaweedfs/values.schema.json

@@ -2,10 +2,21 @@
   "properties": {
     "filer": {
       "properties": {
-        "external": {
-          "default": false,
-          "description": "Enable external access to the SeaweedFS filer from outside the cluster. Use this when `topology` is not set to `Client`.",
-          "type": "boolean"
+        "grpcHost": {
+          "default": "",
+          "description": "The hostname used to expose or access the filer service externally.",
+          "type": "string"
+        },
+        "grpcPort": {
+          "default": 443,
+          "description": "The port used to access the filer service externally.",
+          "type": "number"
+        },
+        "whitelist": {
+          "default": [],
+          "description": "A list of IP addresses or CIDR ranges that are allowed to access the filer service.",
+          "items": {},
+          "type": "array"
         }
       },
       "type": "object"
@@ -15,11 +26,6 @@
       "description": "The hostname used to access the SeaweedFS externally (defaults to 's3' subdomain for the tenant host).",
       "type": "string"
     },
-    "remoteEndpoint": {
-      "default": "1.2.3.4:18888",
-      "description": "The endpoint of the remote filer GRPC service. Used when `topology` is set to `Client`.",
-      "type": "string"
-    },
     "replicas": {
       "default": 2,
       "description": "Persistent Volume size for SeaweedFS",

packages/extra/seaweedfs/values.yaml

@@ -32,9 +32,10 @@ storageClass: ""
 ##     size: 10Gi
 zones: {}
 
-## @param filer.external Enable external access to the SeaweedFS filer from outside the cluster. Use this when `topology` is not set to `Client`.
+## @param filer.grpcHost The hostname used to expose or access the filer service externally.
+## @param filer.grpcPort The port used to access the filer service externally.
+## @param filer.whitelist A list of IP addresses or CIDR ranges that are allowed to access the filer service.
 filer:
-  external: false
-
-## @param remoteEndpoint The endpoint of the remote filer GRPC service. Used when `topology` is set to `Client`.
-remoteEndpoint: "1.2.3.4:18888"
+  grpcHost: ""
+  grpcPort: 443
+  whitelist: []

packages/extra/versions_map

@@ -47,11 +47,13 @@ monitoring 1.9.2 f9f8bb2f
 monitoring 1.10.0 632224a3
 monitoring 1.10.1 8c86905b
 monitoring 1.11.0 4369b031
-monitoring 1.12.0 HEAD
+monitoring 1.12.0 0e47e1e8
+monitoring 1.12.1 HEAD
 seaweedfs 0.1.0 71514249
 seaweedfs 0.2.0 5fb9cfe3
 seaweedfs 0.2.1 fde4bcfa
 seaweedfs 0.3.0 45a7416c
 seaweedfs 0.4.0 632224a3
 seaweedfs 0.4.1 8c86905b
-seaweedfs 0.5.0 HEAD
+seaweedfs 0.5.0 9584e5f5
+seaweedfs 0.6.0 HEAD

packages/system/bucket/images/s3manager.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:34db8c950f14a3e2742b7c31bd3c6a1fe631c9b398caac611ed5cfdac5769d36
+ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:b39a00543d28cb72617bc50c7a9f873d65c3cd34114d9690cf044f7fac66c8da

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/Chart.yaml

@@ -1,11 +1,12 @@
 apiVersion: v2
-appVersion: 0.23.4
+appVersion: 0.25.2
 description: 'Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator).  The
   ClickHouse Operator creates, configures and manages ClickHouse clusters running
   on Kubernetes.  For upgrade please install CRDs separately: ```bash   kubectl apply
-  -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml   kubectl
-  apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml   kubectl
-  apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml
+  -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml   kubectl
+  apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml   kubectl
+  apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml   kubectl
+  apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml
   ```'
 home: https://github.com/Altinity/clickhouse-operator
 icon: https://logosandtypes.com/wp-content/uploads/2020/12/altinity.svg
@@ -14,4 +15,4 @@ maintainers:
   name: altinity
 name: altinity-clickhouse-operator
 type: application
-version: 0.23.4
+version: 0.25.2

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md

@@ -1,6 +1,6 @@
 # altinity-clickhouse-operator
 
-![Version: 0.23.4](https://img.shields.io/badge/Version-0.23.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.23.4](https://img.shields.io/badge/AppVersion-0.23.4-informational?style=flat-square)
+![Version: 0.25.2](https://img.shields.io/badge/Version-0.25.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.25.2](https://img.shields.io/badge/AppVersion-0.25.2-informational?style=flat-square)
 
 Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator).
 
@@ -8,9 +8,10 @@ The ClickHouse Operator creates, configures and manages ClickHouse clusters runn
 
 For upgrade please install CRDs separately:
 ```bash
-  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml
-  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml
-  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml
+  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml
+  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml
+  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml
+  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml
 ```
 
 **Homepage:** <https://github.com/Altinity/clickhouse-operator>
@@ -25,34 +26,38 @@ For upgrade please install CRDs separately:
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| additionalResources | list | `[]` | list of additional resources to create (are processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator, look `kubectl explain chi` for details |
-| affinity | object | `{}` | affinity for scheduler pod assignment, look `kubectl explain pod.spec.affinity` for details |
-| configs | object | check the values.yaml file for the config content, auto-generated from latest operator release | clickhouse-operator configs |
+| additionalResources | list | `[]` | list of additional resources to create (processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator. check `kubectl explain chi` for details |
+| affinity | object | `{}` | affinity for scheduler pod assignment, check `kubectl explain pod.spec.affinity` for details |
+| commonAnnotations | object | `{}` | set of annotations that will be applied to all the resources for the operator |
+| commonLabels | object | `{}` | set of labels that will be applied to all the resources for the operator |
+| configs | object | check the `values.yaml` file for the config content (auto-generated from latest operator release) | clickhouse operator configs |
 | dashboards.additionalLabels | object | `{"grafana_dashboard":""}` | labels to add to a secret with dashboards |
 | dashboards.annotations | object | `{}` | annotations to add to a secret with dashboards |
-| dashboards.enabled | bool | `false` | provision grafana dashboards as secrets (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-6.33.1/charts/grafana/values.yaml#L679 ) |
+| dashboards.enabled | bool | `false` | provision grafana dashboards as configMaps (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-8.3.4/charts/grafana/values.yaml#L778 ) |
 | dashboards.grafana_folder | string | `"clickhouse"` |  |
 | fullnameOverride | string | `""` | full name of the chart. |
-| imagePullSecrets | list | `[]` | image pull secret for private images in clickhouse-operator pod  possible value format [{"name":"your-secret-name"}]  look `kubectl explain pod.spec.imagePullSecrets` for details |
+| imagePullSecrets | list | `[]` | image pull secret for private images in clickhouse-operator pod possible value format `[{"name":"your-secret-name"}]`, check `kubectl explain pod.spec.imagePullSecrets` for details |
 | metrics.containerSecurityContext | object | `{}` |  |
 | metrics.enabled | bool | `true` |  |
-| metrics.env | list | `[]` | additional environment variables for the deployment of metrics-exporter containers possible format value [{"name": "SAMPLE", "value": "text"}] |
+| metrics.env | list | `[]` | additional environment variables for the deployment of metrics-exporter containers possible format value `[{"name": "SAMPLE", "value": "text"}]` |
 | metrics.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
 | metrics.image.repository | string | `"altinity/metrics-exporter"` | image repository |
 | metrics.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) |
 | metrics.resources | object | `{}` | custom resource configuration |
 | nameOverride | string | `""` | override name of the chart |
-| nodeSelector | object | `{}` | node for scheduler pod assignment, look `kubectl explain pod.spec.nodeSelector` for details |
+| namespaceOverride | string | `""` |  |
+| nodeSelector | object | `{}` | node for scheduler pod assignment, check `kubectl explain pod.spec.nodeSelector` for details |
 | operator.containerSecurityContext | object | `{}` |  |
-| operator.env | list | `[]` | additional environment variables for the clickhouse-operator container in deployment possible format value [{"name": "SAMPLE", "value": "text"}] |
+| operator.env | list | `[]` | additional environment variables for the clickhouse-operator container in deployment possible format value `[{"name": "SAMPLE", "value": "text"}]` |
 | operator.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
 | operator.image.repository | string | `"altinity/clickhouse-operator"` | image repository |
 | operator.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) |
-| operator.resources | object | `{}` | custom resource configuration, look `kubectl explain pod.spec.containers.resources` for details |
-| podAnnotations | object | `{"clickhouse-operator-metrics/port":"9999","clickhouse-operator-metrics/scrape":"true","prometheus.io/port":"8888","prometheus.io/scrape":"true"}` | annotations to add to the clickhouse-operator pod, look `kubectl explain pod.spec.annotations` for details |
+| operator.resources | object | `{}` | custom resource configuration, check `kubectl explain pod.spec.containers.resources` for details |
+| podAnnotations | object | check the `values.yaml` file | annotations to add to the clickhouse-operator pod, check `kubectl explain pod.spec.annotations` for details |
 | podLabels | object | `{}` | labels to add to the clickhouse-operator pod |
 | podSecurityContext | object | `{}` |  |
-| rbac.create | bool | `true` | specifies whether cluster roles and cluster role bindings should be created |
+| rbac.create | bool | `true` | specifies whether rbac resources should be created |
+| rbac.namespaceScoped | bool | `false` | specifies whether to create roles and rolebindings at the cluster level or namespace level |
 | secret.create | bool | `true` | create a secret with operator credentials |
 | secret.password | string | `"clickhouse_operator_password"` | operator credentials password |
 | secret.username | string | `"clickhouse_operator"` | operator credentials username |
@@ -60,6 +65,15 @@ For upgrade please install CRDs separately:
 | serviceAccount.create | bool | `true` | specifies whether a service account should be created |
 | serviceAccount.name | string | `nil` | the name of the service account to use; if not set and create is true, a name is generated using the fullname template |
 | serviceMonitor.additionalLabels | object | `{}` | additional labels for service monitor |
-| serviceMonitor.enabled | bool | `false` | ServiceMonitor Custom resource is created for a (prometheus-operator)[https://github.com/prometheus-operator/prometheus-operator] |
-| tolerations | list | `[]` | tolerations for scheduler pod assignment, look `kubectl explain pod.spec.tolerations` for details |
+| serviceMonitor.clickhouseMetrics.interval | string | `"30s"` |  |
+| serviceMonitor.clickhouseMetrics.metricRelabelings | list | `[]` |  |
+| serviceMonitor.clickhouseMetrics.relabelings | list | `[]` |  |
+| serviceMonitor.clickhouseMetrics.scrapeTimeout | string | `""` |  |
+| serviceMonitor.enabled | bool | `false` | ServiceMonitor Custom resource is created for a [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) In serviceMonitor will be created two endpoints clickhouse-metrics on port 8888 and operator-metrics # 9999. Ypu can specify interval, scrapeTimeout, relabelings, metricRelabelings for each endpoint below |
+| serviceMonitor.operatorMetrics.interval | string | `"30s"` |  |
+| serviceMonitor.operatorMetrics.metricRelabelings | list | `[]` |  |
+| serviceMonitor.operatorMetrics.relabelings | list | `[]` |  |
+| serviceMonitor.operatorMetrics.scrapeTimeout | string | `""` |  |
+| tolerations | list | `[]` | tolerations for scheduler pod assignment, check `kubectl explain pod.spec.tolerations` for details |
+| topologySpreadConstraints | list | `[]` |  |
 

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md.gotmpl

@@ -0,0 +1,17 @@
+{{ template "chart.header" . }}
+{{ template "chart.deprecationWarning" . }}
+
+{{ template "chart.badgesSection" . }}
+
+{{ template "chart.description" . }}
+
+{{ template "chart.homepageLine" . }}
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesSection" . }}
+

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml

@@ -4,14 +4,14 @@
 # SINGULAR=clickhouseinstallation
 # PLURAL=clickhouseinstallations
 # SHORT=chi
-# OPERATOR_VERSION=0.23.4
+# OPERATOR_VERSION=0.25.2
 #
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: clickhouseinstallations.clickhouse.altinity.com
   labels:
-    clickhouse.altinity.com/chop: 0.23.4
+    clickhouse.altinity.com/chop: 0.25.2
 spec:
   group: clickhouse.altinity.com
   scope: Namespaced
@@ -51,13 +51,12 @@ spec:
           jsonPath: .status.taskID
         - name: status
           type: string
-          description: CHI status
+          description: Resource status
           jsonPath: .status.status
-        - name: hosts-unchanged
+        - name: hosts-completed
           type: integer
-          description: Unchanged hosts count
-          priority: 1 # show in wide view
-          jsonPath: .status.hostsUnchanged
+          description: Completed hosts count
+          jsonPath: .status.hostsCompleted
         - name: hosts-updated
           type: integer
           description: Updated hosts count
@@ -68,20 +67,11 @@ spec:
           description: Added hosts count
           priority: 1 # show in wide view
           jsonPath: .status.hostsAdded
-        - name: hosts-completed
-          type: integer
-          description: Completed hosts count
-          jsonPath: .status.hostsCompleted
         - name: hosts-deleted
           type: integer
           description: Hosts deleted count
           priority: 1 # show in wide view
           jsonPath: .status.hostsDeleted
-        - name: hosts-delete
-          type: integer
-          description: Hosts to be deleted count
-          priority: 1 # show in wide view
-          jsonPath: .status.hostsDelete
         - name: endpoint
           type: string
           description: Client access endpoint
@@ -92,39 +82,51 @@ spec:
           description: Age of the resource
           # Displayed in all priorities
           jsonPath: .metadata.creationTimestamp
+        - name: suspend
+          type: string
+          description: Suspend reconciliation
+          # Displayed in all priorities
+          jsonPath: .spec.suspend
       subresources:
         status: {}
       schema:
         openAPIV3Schema:
-          description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters"
+          description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters"
           type: object
           required:
             - spec
           properties:
             apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              description: |
+                APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
               type: string
             kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              description: |
+                Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
               type: string
             metadata:
               type: object
             status:
               type: object
-              description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other"
+              description: |
+                Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other
               properties:
                 chop-version:
                   type: string
-                  description: "ClickHouse operator version"
+                  description: "Operator version"
                 chop-commit:
                   type: string
-                  description: "ClickHouse operator git commit SHA"
+                  description: "Operator git commit SHA"
                 chop-date:
                   type: string
-                  description: "ClickHouse operator build date"
+                  description: "Operator build date"
                 chop-ip:
                   type: string
-                  description: "IP address of the operator's pod which managed this CHI"
+                  description: "IP address of the operator's pod which managed this resource"
                 clusters:
                   type: integer
                   minimum: 0
@@ -222,17 +224,23 @@ spec:
                 endpoint:
                   type: string
                   description: "Endpoint"
+                endpoints:
+                  type: array
+                  description: "All endpoints"
+                  nullable: true
+                  items:
+                    type: string
                 generation:
                   type: integer
                   minimum: 0
                   description: "Generation"
                 normalized:
                   type: object
-                  description: "Normalized CHI requested"
+                  description: "Normalized resource requested"
                   x-kubernetes-preserve-unknown-fields: true
                 normalizedCompleted:
                   type: object
-                  description: "Normalized CHI completed"
+                  description: "Normalized resource completed"
                   x-kubernetes-preserve-unknown-fields: true
                 hostsWithTablesCreated:
                   type: array
@@ -240,6 +248,12 @@ spec:
                   nullable: true
                   items:
                     type: string
+                hostsWithReplicaCaughtUp:
+                  type: array
+                  description: "List of hosts with replica caught up"
+                  nullable: true
+                  items:
+                    type: string
                 usedTemplates:
                   type: array
                   description: "List of templates used to build this CHI"
@@ -301,6 +315,13 @@ spec:
                   enum:
                     - ""
                     - "RollingUpdate"
+                suspend:
+                  !!merge <<: *TypeStringBool
+                  description: |
+                    Suspend reconciliation of resources managed by a ClickHouse Installation.
+                    Works as the following:
+                     - When `suspend` is `true` operator stops reconciling all resources.
+                     - When `suspend` is `false` or not set, operator reconciles all resources.
                 troubleshoot:
                   !!merge <<: *TypeStringBool
                   description: |
@@ -412,6 +433,63 @@ spec:
                             service:
                               !!merge <<: *TypeObjectsCleanup
                               description: "Behavior policy for failed Service, `Retain` by default"
+                    runtime:
+                      type: object
+                      description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle"
+                      properties:
+                        reconcileShardsThreadsNumber:
+                          type: integer
+                          minimum: 1
+                          maximum: 65535
+                          description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default"
+                        reconcileShardsMaxConcurrencyPercent:
+                          type: integer
+                          minimum: 0
+                          maximum: 100
+                          description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default."
+                    macros:
+                      type: object
+                      description: "macros parameters"
+                      properties:
+                        sections:
+                          type: object
+                          description: "sections behaviour for macros"
+                          properties:
+                            users:
+                              type: object
+                              description: "sections behaviour for macros on users"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
+                            profiles:
+                              type: object
+                              description: "sections behaviour for macros on profiles"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
+                            quotas:
+                              type: object
+                              description: "sections behaviour for macros on quotas"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
+                            settings:
+                              type: object
+                              description: "sections behaviour for macros on settings"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
+                            files:
+                              type: object
+                              description: "sections behaviour for macros on files"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
                 defaults:
                   type: object
                   description: |
@@ -424,7 +502,7 @@ spec:
                       description: |
                         define should replicas be specified by FQDN in `<host></host>`.
                         In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup
-                        "yes" by default
+                        "no" by default
                     distributedDDL:
                       type: object
                       description: |
@@ -474,7 +552,13 @@ spec:
                           description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`"
                         serviceTemplate:
                           type: string
-                          description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource"
+                          description: "optional, template name from chi.spec.templates.serviceTemplates. used for customization of the `Service` resource, created by `clickhouse-operator` to cover all clusters in whole `chi` resource"
+                        serviceTemplates:
+                          type: array
+                          description: "optional, template names from chi.spec.templates.serviceTemplates. used for customization of the `Service` resources, created by `clickhouse-operator` to cover all clusters in whole `chi` resource"
+                          nullable: true
+                          items:
+                            type: string
                         clusterServiceTemplate:
                           type: string
                           description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`"
@@ -486,7 +570,7 @@ spec:
                           description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`"
                         volumeClaimTemplate:
                           type: string
-                          description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate"
+                          description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`"
                 configuration:
                   type: object
                   description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource"
@@ -521,6 +605,9 @@ spec:
                               secure:
                                 !!merge <<: *TypeStringBool
                                 description: "if a secure connection to Zookeeper is required"
+                              availabilityZone:
+                                type: string
+                                description: "availability zone for Zookeeper node"
                         session_timeout_ms:
                           type: integer
                           description: "session timeout during connect to Zookeeper"
@@ -540,6 +627,20 @@ spec:
                         you can configure password hashed, authorization restrictions, database level security row filters etc.
                         More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/
                         Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers
+
+                        any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
+                        secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
+                        it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
+
+                        look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples
+
+                        any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key
+                        in this case value from secret will write directly into XML tag during render *-usersd ConfigMap
+
+                        any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key
+                        in this case value from secret will write into environment variable and write to XML tag via from_env=XXX
+
+                        look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples
                       # nullable: true
                       x-kubernetes-preserve-unknown-fields: true
                     profiles:
@@ -566,6 +667,12 @@ spec:
                         allows configure `clickhouse-server` settings inside <yandex>...</yandex> tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/`
                         More details: https://clickhouse.tech/docs/en/operations/settings/settings/
                         Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings
+
+                        any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
+                        look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples
+
+                        secret value will pass in `pod.spec.env`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml
+                        it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
                       # nullable: true
                       x-kubernetes-preserve-unknown-fields: true
                     files: &TypeFiles
@@ -575,14 +682,20 @@ spec:
                         every key in this object is the file name
                         every value in this object is the file content
                         you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html
-                        each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored
+                        each key could contains prefix like {common}, {users}, {hosts} or config.d, users.d, conf.d, wrong prefixes will be ignored, subfolders also will be ignored
                         More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml
+
+                        any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets
+                        secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/
+                        and will automatically update when update secret
+                        it useful for pass SSL certificates from cert-manager or similar tool
+                        look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples
                       # nullable: true
                       x-kubernetes-preserve-unknown-fields: true
                     clusters:
                       type: array
                       description: |
-                        describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level
+                        describes clusters layout and allows change settings on cluster-level, shard-level and replica-level
                         every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server`
                         all Pods will rendered in <remote_server> part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml`
                         Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/
@@ -595,7 +708,7 @@ spec:
                         properties:
                           name:
                             type: string
-                            description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources"
+                            description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources"
                             minLength: 1
                             # See namePartClusterMaxLen const
                             maxLength: 15
@@ -683,6 +796,32 @@ spec:
                                     required:
                                       - name
                                       - key
+                          pdbMaxUnavailable:
+                            type: integer
+                            description: |
+                              Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction,
+                              i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions
+                              by specifying 0. This is a mutually exclusive setting with "minAvailable".
+                            minimum: 0
+                            maximum: 65535
+                          reconcile:
+                            type: object
+                            description: "allow tuning reconciling process"
+                            properties:
+                              runtime:
+                                type: object
+                                description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle"
+                                properties:
+                                  reconcileShardsThreadsNumber:
+                                    type: integer
+                                    minimum: 1
+                                    maximum: 65535
+                                    description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default"
+                                  reconcileShardsMaxConcurrencyPercent:
+                                    type: integer
+                                    minimum: 0
+                                    maximum: 100
+                                    description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default."
                           layout:
                             type: object
                             description: |
@@ -690,18 +829,24 @@ spec:
                               allows override settings on each shard and replica separatelly
                             # nullable: true
                             properties:
-                              type:
-                                type: string
-                                description: "DEPRECATED - to be removed soon"
                               shardsCount:
                                 type: integer
-                                description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default"
+                                description: |
+                                  how much shards for current ClickHouse cluster will run in Kubernetes,
+                                  each shard contains shared-nothing part of data and contains set of replicas,
+                                  cluster contains 1 shard by default"
                               replicasCount:
                                 type: integer
-                                description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default"
+                                description: |
+                                  how much replicas in each shards for current cluster will run in Kubernetes,
+                                  each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance,
+                                  every shard contains 1 replica by default"
                               shards:
                                 type: array
-                                description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do"
+                                description: |
+                                  optional, allows override top-level `chi.spec.configuration`, cluster-level
+                                  `chi.spec.configuration.clusters` settings for each shard separately,
+                                  use it only if you fully understand what you do"
                                 # nullable: true
                                 items:
                                   type: object
@@ -1036,7 +1181,7 @@ spec:
                             description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`"
                           generateName:
                             type: string
-                            description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables"
+                            description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables"
                           zone:
                             type: object
                             description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`"
@@ -1108,7 +1253,9 @@ spec:
                                   maximum: 65535
                                 topologyKey:
                                   type: string
-                                  description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity"
+                                  description: |
+                                    use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`,
+                                    more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity"
                           metadata:
                             type: object
                             description: |
@@ -1124,7 +1271,8 @@ spec:
                             x-kubernetes-preserve-unknown-fields: true
                     volumeClaimTemplates:
                       type: array
-                      description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else"
+                      description: |
+                        allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else
                       # nullable: true
                       items:
                         type: object
@@ -1177,14 +1325,17 @@ spec:
                               replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate`
                           generateName:
                             type: string
-                            description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables"
+                            description: |
+                              allows define format for generated `Service` name,
+                              look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates
+                              for details about available template variables"
                           metadata:
                             # TODO specify ObjectMeta
                             type: object
                             description: |
                               allows pass standard object's metadata from template to Service
                               Could be use for define specificly for Cloud Provider metadata which impact to behavior of service
-                              More info: https://kubernetes.io/docs/concepts/services-networking/service/
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
                             # nullable: true
                             x-kubernetes-preserve-unknown-fields: true
                           spec:
@@ -1197,7 +1348,9 @@ spec:
                             x-kubernetes-preserve-unknown-fields: true
                 useTemplates:
                   type: array
-                  description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters"
+                  description: |
+                    list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI`
+                    manifest during render Kubernetes resources to create related ClickHouse clusters"
                   # nullable: true
                   items:
                     type: object

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml

@@ -4,14 +4,14 @@
 # SINGULAR=clickhouseinstallationtemplate
 # PLURAL=clickhouseinstallationtemplates
 # SHORT=chit
-# OPERATOR_VERSION=0.23.4
+# OPERATOR_VERSION=0.25.2
 #
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: clickhouseinstallationtemplates.clickhouse.altinity.com
   labels:
-    clickhouse.altinity.com/chop: 0.23.4
+    clickhouse.altinity.com/chop: 0.25.2
 spec:
   group: clickhouse.altinity.com
   scope: Namespaced
@@ -51,13 +51,12 @@ spec:
           jsonPath: .status.taskID
         - name: status
           type: string
-          description: CHI status
+          description: Resource status
           jsonPath: .status.status
-        - name: hosts-unchanged
+        - name: hosts-completed
           type: integer
-          description: Unchanged hosts count
-          priority: 1 # show in wide view
-          jsonPath: .status.hostsUnchanged
+          description: Completed hosts count
+          jsonPath: .status.hostsCompleted
         - name: hosts-updated
           type: integer
           description: Updated hosts count
@@ -68,20 +67,11 @@ spec:
           description: Added hosts count
           priority: 1 # show in wide view
           jsonPath: .status.hostsAdded
-        - name: hosts-completed
-          type: integer
-          description: Completed hosts count
-          jsonPath: .status.hostsCompleted
         - name: hosts-deleted
           type: integer
           description: Hosts deleted count
           priority: 1 # show in wide view
           jsonPath: .status.hostsDeleted
-        - name: hosts-delete
-          type: integer
-          description: Hosts to be deleted count
-          priority: 1 # show in wide view
-          jsonPath: .status.hostsDelete
         - name: endpoint
           type: string
           description: Client access endpoint
@@ -92,39 +82,51 @@ spec:
           description: Age of the resource
           # Displayed in all priorities
           jsonPath: .metadata.creationTimestamp
+        - name: suspend
+          type: string
+          description: Suspend reconciliation
+          # Displayed in all priorities
+          jsonPath: .spec.suspend
       subresources:
         status: {}
       schema:
         openAPIV3Schema:
-          description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters"
+          description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters"
           type: object
           required:
             - spec
           properties:
             apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              description: |
+                APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
               type: string
             kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              description: |
+                Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
               type: string
             metadata:
               type: object
             status:
               type: object
-              description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other"
+              description: |
+                Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other
               properties:
                 chop-version:
                   type: string
-                  description: "ClickHouse operator version"
+                  description: "Operator version"
                 chop-commit:
                   type: string
-                  description: "ClickHouse operator git commit SHA"
+                  description: "Operator git commit SHA"
                 chop-date:
                   type: string
-                  description: "ClickHouse operator build date"
+                  description: "Operator build date"
                 chop-ip:
                   type: string
-                  description: "IP address of the operator's pod which managed this CHI"
+                  description: "IP address of the operator's pod which managed this resource"
                 clusters:
                   type: integer
                   minimum: 0
@@ -222,17 +224,23 @@ spec:
                 endpoint:
                   type: string
                   description: "Endpoint"
+                endpoints:
+                  type: array
+                  description: "All endpoints"
+                  nullable: true
+                  items:
+                    type: string
                 generation:
                   type: integer
                   minimum: 0
                   description: "Generation"
                 normalized:
                   type: object
-                  description: "Normalized CHI requested"
+                  description: "Normalized resource requested"
                   x-kubernetes-preserve-unknown-fields: true
                 normalizedCompleted:
                   type: object
-                  description: "Normalized CHI completed"
+                  description: "Normalized resource completed"
                   x-kubernetes-preserve-unknown-fields: true
                 hostsWithTablesCreated:
                   type: array
@@ -240,6 +248,12 @@ spec:
                   nullable: true
                   items:
                     type: string
+                hostsWithReplicaCaughtUp:
+                  type: array
+                  description: "List of hosts with replica caught up"
+                  nullable: true
+                  items:
+                    type: string
                 usedTemplates:
                   type: array
                   description: "List of templates used to build this CHI"
@@ -301,6 +315,13 @@ spec:
                   enum:
                     - ""
                     - "RollingUpdate"
+                suspend:
+                  !!merge <<: *TypeStringBool
+                  description: |
+                    Suspend reconciliation of resources managed by a ClickHouse Installation.
+                    Works as the following:
+                     - When `suspend` is `true` operator stops reconciling all resources.
+                     - When `suspend` is `false` or not set, operator reconciles all resources.
                 troubleshoot:
                   !!merge <<: *TypeStringBool
                   description: |
@@ -412,6 +433,63 @@ spec:
                             service:
                               !!merge <<: *TypeObjectsCleanup
                               description: "Behavior policy for failed Service, `Retain` by default"
+                    runtime:
+                      type: object
+                      description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle"
+                      properties:
+                        reconcileShardsThreadsNumber:
+                          type: integer
+                          minimum: 1
+                          maximum: 65535
+                          description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default"
+                        reconcileShardsMaxConcurrencyPercent:
+                          type: integer
+                          minimum: 0
+                          maximum: 100
+                          description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default."
+                    macros:
+                      type: object
+                      description: "macros parameters"
+                      properties:
+                        sections:
+                          type: object
+                          description: "sections behaviour for macros"
+                          properties:
+                            users:
+                              type: object
+                              description: "sections behaviour for macros on users"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
+                            profiles:
+                              type: object
+                              description: "sections behaviour for macros on profiles"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
+                            quotas:
+                              type: object
+                              description: "sections behaviour for macros on quotas"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
+                            settings:
+                              type: object
+                              description: "sections behaviour for macros on settings"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
+                            files:
+                              type: object
+                              description: "sections behaviour for macros on files"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "enabled or not"
                 defaults:
                   type: object
                   description: |
@@ -424,7 +502,7 @@ spec:
                       description: |
                         define should replicas be specified by FQDN in `<host></host>`.
                         In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup
-                        "yes" by default
+                        "no" by default
                     distributedDDL:
                       type: object
                       description: |
@@ -474,7 +552,13 @@ spec:
                           description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`"
                         serviceTemplate:
                           type: string
-                          description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource"
+                          description: "optional, template name from chi.spec.templates.serviceTemplates. used for customization of the `Service` resource, created by `clickhouse-operator` to cover all clusters in whole `chi` resource"
+                        serviceTemplates:
+                          type: array
+                          description: "optional, template names from chi.spec.templates.serviceTemplates. used for customization of the `Service` resources, created by `clickhouse-operator` to cover all clusters in whole `chi` resource"
+                          nullable: true
+                          items:
+                            type: string
                         clusterServiceTemplate:
                           type: string
                           description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`"
@@ -486,7 +570,7 @@ spec:
                           description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`"
                         volumeClaimTemplate:
                           type: string
-                          description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate"
+                          description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`"
                 configuration:
                   type: object
                   description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource"
@@ -521,6 +605,9 @@ spec:
                               secure:
                                 !!merge <<: *TypeStringBool
                                 description: "if a secure connection to Zookeeper is required"
+                              availabilityZone:
+                                type: string
+                                description: "availability zone for Zookeeper node"
                         session_timeout_ms:
                           type: integer
                           description: "session timeout during connect to Zookeeper"
@@ -540,6 +627,20 @@ spec:
                         you can configure password hashed, authorization restrictions, database level security row filters etc.
                         More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/
                         Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers
+
+                        any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
+                        secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
+                        it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
+
+                        look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples
+
+                        any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key
+                        in this case value from secret will write directly into XML tag during render *-usersd ConfigMap
+
+                        any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key
+                        in this case value from secret will write into environment variable and write to XML tag via from_env=XXX
+
+                        look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples
                       # nullable: true
                       x-kubernetes-preserve-unknown-fields: true
                     profiles:
@@ -566,6 +667,12 @@ spec:
                         allows configure `clickhouse-server` settings inside <yandex>...</yandex> tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/`
                         More details: https://clickhouse.tech/docs/en/operations/settings/settings/
                         Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings
+
+                        any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
+                        look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples
+
+                        secret value will pass in `pod.spec.env`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml
+                        it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
                       # nullable: true
                       x-kubernetes-preserve-unknown-fields: true
                     files: &TypeFiles
@@ -575,14 +682,20 @@ spec:
                         every key in this object is the file name
                         every value in this object is the file content
                         you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html
-                        each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored
+                        each key could contains prefix like {common}, {users}, {hosts} or config.d, users.d, conf.d, wrong prefixes will be ignored, subfolders also will be ignored
                         More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml
+
+                        any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets
+                        secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/
+                        and will automatically update when update secret
+                        it useful for pass SSL certificates from cert-manager or similar tool
+                        look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples
                       # nullable: true
                       x-kubernetes-preserve-unknown-fields: true
                     clusters:
                       type: array
                       description: |
-                        describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level
+                        describes clusters layout and allows change settings on cluster-level, shard-level and replica-level
                         every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server`
                         all Pods will rendered in <remote_server> part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml`
                         Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/
@@ -595,7 +708,7 @@ spec:
                         properties:
                           name:
                             type: string
-                            description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources"
+                            description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources"
                             minLength: 1
                             # See namePartClusterMaxLen const
                             maxLength: 15
@@ -683,6 +796,32 @@ spec:
                                     required:
                                       - name
                                       - key
+                          pdbMaxUnavailable:
+                            type: integer
+                            description: |
+                              Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction,
+                              i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions
+                              by specifying 0. This is a mutually exclusive setting with "minAvailable".
+                            minimum: 0
+                            maximum: 65535
+                          reconcile:
+                            type: object
+                            description: "allow tuning reconciling process"
+                            properties:
+                              runtime:
+                                type: object
+                                description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle"
+                                properties:
+                                  reconcileShardsThreadsNumber:
+                                    type: integer
+                                    minimum: 1
+                                    maximum: 65535
+                                    description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default"
+                                  reconcileShardsMaxConcurrencyPercent:
+                                    type: integer
+                                    minimum: 0
+                                    maximum: 100
+                                    description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default."
                           layout:
                             type: object
                             description: |
@@ -690,18 +829,24 @@ spec:
                               allows override settings on each shard and replica separatelly
                             # nullable: true
                             properties:
-                              type:
-                                type: string
-                                description: "DEPRECATED - to be removed soon"
                               shardsCount:
                                 type: integer
-                                description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default"
+                                description: |
+                                  how much shards for current ClickHouse cluster will run in Kubernetes,
+                                  each shard contains shared-nothing part of data and contains set of replicas,
+                                  cluster contains 1 shard by default"
                               replicasCount:
                                 type: integer
-                                description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default"
+                                description: |
+                                  how much replicas in each shards for current cluster will run in Kubernetes,
+                                  each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance,
+                                  every shard contains 1 replica by default"
                               shards:
                                 type: array
-                                description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do"
+                                description: |
+                                  optional, allows override top-level `chi.spec.configuration`, cluster-level
+                                  `chi.spec.configuration.clusters` settings for each shard separately,
+                                  use it only if you fully understand what you do"
                                 # nullable: true
                                 items:
                                   type: object
@@ -1036,7 +1181,7 @@ spec:
                             description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`"
                           generateName:
                             type: string
-                            description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables"
+                            description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables"
                           zone:
                             type: object
                             description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`"
@@ -1108,7 +1253,9 @@ spec:
                                   maximum: 65535
                                 topologyKey:
                                   type: string
-                                  description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity"
+                                  description: |
+                                    use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`,
+                                    more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity"
                           metadata:
                             type: object
                             description: |
@@ -1124,7 +1271,8 @@ spec:
                             x-kubernetes-preserve-unknown-fields: true
                     volumeClaimTemplates:
                       type: array
-                      description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else"
+                      description: |
+                        allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else
                       # nullable: true
                       items:
                         type: object
@@ -1177,14 +1325,17 @@ spec:
                               replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate`
                           generateName:
                             type: string
-                            description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables"
+                            description: |
+                              allows define format for generated `Service` name,
+                              look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates
+                              for details about available template variables"
                           metadata:
                             # TODO specify ObjectMeta
                             type: object
                             description: |
                               allows pass standard object's metadata from template to Service
                               Could be use for define specificly for Cloud Provider metadata which impact to behavior of service
-                              More info: https://kubernetes.io/docs/concepts/services-networking/service/
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
                             # nullable: true
                             x-kubernetes-preserve-unknown-fields: true
                           spec:
@@ -1197,7 +1348,9 @@ spec:
                             x-kubernetes-preserve-unknown-fields: true
                 useTemplates:
                   type: array
-                  description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters"
+                  description: |
+                    list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI`
+                    manifest during render Kubernetes resources to create related ClickHouse clusters"
                   # nullable: true
                   items:
                     type: object

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml

@@ -1,13 +1,13 @@
 # Template Parameters:
 #
-# OPERATOR_VERSION=0.23.4
+# OPERATOR_VERSION=0.25.2
 #
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com
   labels:
-    clickhouse-keeper.altinity.com/chop: 0.23.4
+    clickhouse-keeper.altinity.com/chop: 0.25.2
 spec:
   group: clickhouse-keeper.altinity.com
   scope: Namespaced
@@ -22,123 +22,487 @@ spec:
       served: true
       storage: true
       additionalPrinterColumns:
+        - name: version
+          type: string
+          description: Operator version
+          priority: 1 # show in wide view
+          jsonPath: .status.chop-version
+        - name: clusters
+          type: integer
+          description: Clusters count
+          jsonPath: .status.clusters
+        - name: shards
+          type: integer
+          description: Shards count
+          priority: 1 # show in wide view
+          jsonPath: .status.shards
+        - name: hosts
+          type: integer
+          description: Hosts count
+          jsonPath: .status.hosts
+        - name: taskID
+          type: string
+          description: TaskID
+          priority: 1 # show in wide view
+          jsonPath: .status.taskID
         - name: status
           type: string
-          description: CHK status
+          description: Resource status
           jsonPath: .status.status
-        - name: replicas
+        - name: hosts-unchanged
           type: integer
-          description: Replica count
+          description: Unchanged hosts count
           priority: 1 # show in wide view
-          jsonPath: .status.replicas
+          jsonPath: .status.hostsUnchanged
+        - name: hosts-updated
+          type: integer
+          description: Updated hosts count
+          priority: 1 # show in wide view
+          jsonPath: .status.hostsUpdated
+        - name: hosts-added
+          type: integer
+          description: Added hosts count
+          priority: 1 # show in wide view
+          jsonPath: .status.hostsAdded
+        - name: hosts-completed
+          type: integer
+          description: Completed hosts count
+          jsonPath: .status.hostsCompleted
+        - name: hosts-deleted
+          type: integer
+          description: Hosts deleted count
+          priority: 1 # show in wide view
+          jsonPath: .status.hostsDeleted
+        - name: hosts-delete
+          type: integer
+          description: Hosts to be deleted count
+          priority: 1 # show in wide view
+          jsonPath: .status.hostsDelete
+        - name: endpoint
+          type: string
+          description: Client access endpoint
+          priority: 1 # show in wide view
+          jsonPath: .status.endpoint
         - name: age
           type: date
           description: Age of the resource
           # Displayed in all priorities
           jsonPath: .metadata.creationTimestamp
+        - name: suspend
+          type: string
+          description: Suspend reconciliation
+          # Displayed in all priorities
+          jsonPath: .spec.suspend
       subresources:
         status: {}
       schema:
         openAPIV3Schema:
+          description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters"
           type: object
           required:
             - spec
-          description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one ClickHouse Keeper cluster"
           properties:
             apiVersion:
-              type: string
               description: |
                 APIVersion defines the versioned schema of this representation
                 of an object. Servers should convert recognized schemas to the latest
                 internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
-            kind:
               type: string
+            kind:
               description: |
                 Kind is a string value representing the REST resource this
                 object represents. Servers may infer this from the endpoint the client
                 submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
             metadata:
               type: object
             status:
               type: object
               description: |
-                Current ClickHouseKeeperInstallation status, contains many fields like overall status, desired replicas and ready replica list with their endpoints
+                Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other
               properties:
                 chop-version:
                   type: string
-                  description: "ClickHouse operator version"
+                  description: "Operator version"
                 chop-commit:
                   type: string
-                  description: "ClickHouse operator git commit SHA"
+                  description: "Operator git commit SHA"
                 chop-date:
                   type: string
-                  description: "ClickHouse operator build date"
+                  description: "Operator build date"
                 chop-ip:
                   type: string
-                  description: "IP address of the operator's pod which managed this CHI"
+                  description: "IP address of the operator's pod which managed this resource"
+                clusters:
+                  type: integer
+                  minimum: 0
+                  description: "Clusters count"
+                shards:
+                  type: integer
+                  minimum: 0
+                  description: "Shards count"
+                replicas:
+                  type: integer
+                  minimum: 0
+                  description: "Replicas count"
+                hosts:
+                  type: integer
+                  minimum: 0
+                  description: "Hosts count"
                 status:
                   type: string
                   description: "Status"
-                replicas:
-                  type: integer
-                  format: int32
-                  description: Replicas is the number of number of desired replicas in the cluster
-                readyReplicas:
+                taskID:
+                  type: string
+                  description: "Current task id"
+                taskIDsStarted:
                   type: array
-                  description: ReadyReplicas is the array of endpoints of those ready replicas in the cluster
+                  description: "Started task ids"
+                  nullable: true
                   items:
-                    type: object
-                    properties:
-                      host:
-                        type: string
-                        description: dns name or ip address for Keeper node
-                      port:
-                        type: integer
-                        minimum: 0
-                        maximum: 65535
-                        description: TCP port which used to connect to Keeper node
-                      secure:
-                        type: string
-                        description: if a secure connection to Keeper is required
+                    type: string
+                taskIDsCompleted:
+                  type: array
+                  description: "Completed task ids"
+                  nullable: true
+                  items:
+                    type: string
+                action:
+                  type: string
+                  description: "Action"
+                actions:
+                  type: array
+                  description: "Actions"
+                  nullable: true
+                  items:
+                    type: string
+                error:
+                  type: string
+                  description: "Last error"
+                errors:
+                  type: array
+                  description: "Errors"
+                  nullable: true
+                  items:
+                    type: string
+                hostsUnchanged:
+                  type: integer
+                  minimum: 0
+                  description: "Unchanged Hosts count"
+                hostsUpdated:
+                  type: integer
+                  minimum: 0
+                  description: "Updated Hosts count"
+                hostsAdded:
+                  type: integer
+                  minimum: 0
+                  description: "Added Hosts count"
+                hostsCompleted:
+                  type: integer
+                  minimum: 0
+                  description: "Completed Hosts count"
+                hostsDeleted:
+                  type: integer
+                  minimum: 0
+                  description: "Deleted Hosts count"
+                hostsDelete:
+                  type: integer
+                  minimum: 0
+                  description: "About to delete Hosts count"
+                pods:
+                  type: array
+                  description: "Pods"
+                  nullable: true
+                  items:
+                    type: string
+                pod-ips:
+                  type: array
+                  description: "Pod IPs"
+                  nullable: true
+                  items:
+                    type: string
+                fqdns:
+                  type: array
+                  description: "Pods FQDNs"
+                  nullable: true
+                  items:
+                    type: string
+                endpoint:
+                  type: string
+                  description: "Endpoint"
+                endpoints:
+                  type: array
+                  description: "All endpoints"
+                  nullable: true
+                  items:
+                    type: string
+                generation:
+                  type: integer
+                  minimum: 0
+                  description: "Generation"
                 normalized:
                   type: object
-                  description: "Normalized CHK requested"
+                  description: "Normalized resource requested"
                   x-kubernetes-preserve-unknown-fields: true
                 normalizedCompleted:
                   type: object
-                  description: "Normalized CHK completed"
+                  description: "Normalized resource completed"
                   x-kubernetes-preserve-unknown-fields: true
+                hostsWithTablesCreated:
+                  type: array
+                  description: "List of hosts with tables created by the operator"
+                  nullable: true
+                  items:
+                    type: string
+                hostsWithReplicaCaughtUp:
+                  type: array
+                  description: "List of hosts with replica caught up"
+                  nullable: true
+                  items:
+                    type: string
+                usedTemplates:
+                  type: array
+                  description: "List of templates used to build this CHI"
+                  nullable: true
+                  x-kubernetes-preserve-unknown-fields: true
+                  items:
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
             spec:
               type: object
-              description: KeeperSpec defines the desired state of a Keeper cluster
+              # x-kubernetes-preserve-unknown-fields: true
+              description: |
+                Specification of the desired behavior of one or more ClickHouse clusters
+                More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md
               properties:
+                taskID:
+                  type: string
+                  description: |
+                    Allows to define custom taskID for CHI update and watch status of this update execution.
+                    Displayed in all .status.taskID* fields.
+                    By default (if not filled) every update of CHI manifest will generate random taskID
+                stop: &TypeStringBool
+                  type: string
+                  description: |
+                    Allows to stop all ClickHouse clusters defined in a CHI.
+                    Works as the following:
+                     - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact.
+                     - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s.
+                  enum:
+                    # List StringBoolXXX constants from model
+                    - ""
+                    - "0"
+                    - "1"
+                    - "False"
+                    - "false"
+                    - "True"
+                    - "true"
+                    - "No"
+                    - "no"
+                    - "Yes"
+                    - "yes"
+                    - "Off"
+                    - "off"
+                    - "On"
+                    - "on"
+                    - "Disable"
+                    - "disable"
+                    - "Enable"
+                    - "enable"
+                    - "Disabled"
+                    - "disabled"
+                    - "Enabled"
+                    - "enabled"
+                suspend:
+                  !!merge <<: *TypeStringBool
+                  description: |
+                    Suspend reconciliation of resources managed by a ClickHouse Keeper.
+                    Works as the following:
+                     - When `suspend` is `true` operator stops reconciling all resources.
+                     - When `suspend` is `false` or not set, operator reconciles all resources.
                 namespaceDomainPattern:
                   type: string
                   description: |
                     Custom domain pattern which will be used for DNS names of `Service` or `Pod`.
                     Typical use scenario - custom cluster domain in Kubernetes cluster
                     Example: %s.svc.my.test
-                replicas:
-                  type: integer
-                  format: int32
+                reconciling:
+                  type: object
+                  description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side"
+                  # nullable: true
+                  properties:
+                    policy:
+                      type: string
+                      description: |
+                        DISCUSSED TO BE DEPRECATED
+                        Syntax sugar
+                        Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config
+                        Possible values:
+                         - wait - should wait to exclude host, complete queries and include host back into the cluster
+                         - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster
+                      enum:
+                        - ""
+                        - "wait"
+                        - "nowait"
+                    configMapPropagationTimeout:
+                      type: integer
+                      description: |
+                        Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod`
+                        More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically
+                      minimum: 0
+                      maximum: 3600
+                    cleanup:
+                      type: object
+                      description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle"
+                      # nullable: true
+                      properties:
+                        unknownObjects:
+                          type: object
+                          description: |
+                            Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator,
+                            but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource.
+                            Default behavior is `Delete`"
+                          # nullable: true
+                          properties:
+                            statefulSet: &TypeObjectsCleanup
+                              type: string
+                              description: "Behavior policy for unknown StatefulSet, `Delete` by default"
+                              enum:
+                                # List ObjectsCleanupXXX constants from model
+                                - ""
+                                - "Retain"
+                                - "Delete"
+                            pvc:
+                              type: string
+                              !!merge <<: *TypeObjectsCleanup
+                              description: "Behavior policy for unknown PVC, `Delete` by default"
+                            configMap:
+                              !!merge <<: *TypeObjectsCleanup
+                              description: "Behavior policy for unknown ConfigMap, `Delete` by default"
+                            service:
+                              !!merge <<: *TypeObjectsCleanup
+                              description: "Behavior policy for unknown Service, `Delete` by default"
+                        reconcileFailedObjects:
+                          type: object
+                          description: |
+                            Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile.
+                            Default behavior is `Retain`"
+                          # nullable: true
+                          properties:
+                            statefulSet:
+                              !!merge <<: *TypeObjectsCleanup
+                              description: "Behavior policy for failed StatefulSet, `Retain` by default"
+                            pvc:
+                              !!merge <<: *TypeObjectsCleanup
+                              description: "Behavior policy for failed PVC, `Retain` by default"
+                            configMap:
+                              !!merge <<: *TypeObjectsCleanup
+                              description: "Behavior policy for failed ConfigMap, `Retain` by default"
+                            service:
+                              !!merge <<: *TypeObjectsCleanup
+                              description: "Behavior policy for failed Service, `Retain` by default"
+                defaults:
+                  type: object
                   description: |
-                    Replicas is the expected size of the keeper cluster.
-                    The valid range of size is from 1 to 7.
-                  minimum: 1
-                  maximum: 7
+                    define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level
+                    More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults
+                  # nullable: true
+                  properties:
+                    replicasUseFQDN:
+                      !!merge <<: *TypeStringBool
+                      description: |
+                        define should replicas be specified by FQDN in `<host></host>`.
+                        In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup
+                        "no" by default
+                    distributedDDL:
+                      type: object
+                      description: |
+                        allows change `<yandex><distributed_ddl></distributed_ddl></yandex>` settings
+                        More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl
+                      # nullable: true
+                      properties:
+                        profile:
+                          type: string
+                          description: "Settings from this profile will be used to execute DDL queries"
+                    storageManagement:
+                      type: object
+                      description: default storage management options
+                      properties:
+                        provisioner: &TypePVCProvisioner
+                          type: string
+                          description: "defines `PVC` provisioner - be it StatefulSet or the Operator"
+                          enum:
+                            - ""
+                            - "StatefulSet"
+                            - "Operator"
+                        reclaimPolicy: &TypePVCReclaimPolicy
+                          type: string
+                          description: |
+                            defines behavior of `PVC` deletion.
+                            `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet
+                          enum:
+                            - ""
+                            - "Retain"
+                            - "Delete"
+                    templates: &TypeTemplateNames
+                      type: object
+                      description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource"
+                      # nullable: true
+                      properties:
+                        hostTemplate:
+                          type: string
+                          description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`"
+                        podTemplate:
+                          type: string
+                          description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`"
+                        dataVolumeClaimTemplate:
+                          type: string
+                          description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`"
+                        logVolumeClaimTemplate:
+                          type: string
+                          description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`"
+                        serviceTemplate:
+                          type: string
+                          description: "optional, template name from chi.spec.templates.serviceTemplates. used for customization of the `Service` resource, created by `clickhouse-operator` to cover all clusters in whole `chi` resource"
+                        serviceTemplates:
+                          type: array
+                          description: "optional, template names from chi.spec.templates.serviceTemplates. used for customization of the `Service` resources, created by `clickhouse-operator` to cover all clusters in whole `chi` resource"
+                          nullable: true
+                          items:
+                            type: string
+                        clusterServiceTemplate:
+                          type: string
+                          description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`"
+                        shardServiceTemplate:
+                          type: string
+                          description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`"
+                        replicaServiceTemplate:
+                          type: string
+                          description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`"
+                        volumeClaimTemplate:
+                          type: string
+                          description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`"
                 configuration:
                   type: object
                   description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource"
                   # nullable: true
                   properties:
-                    settings:
+                    settings: &TypeSettings
                       type: object
-                      description: "allows configure multiple aspects and behavior for `clickhouse-keeper` instance"
+                      description: |
+                        allows configure multiple aspects and behavior for `clickhouse-keeper` instance
+                      # nullable: true
+                      x-kubernetes-preserve-unknown-fields: true
+                    files: &TypeFiles
+                      type: object
+                      description: |
+                        allows define content of any setting
+                      # nullable: true
                       x-kubernetes-preserve-unknown-fields: true
                     clusters:
                       type: array
                       description: |
-                        describes ClickHouseKeeper clusters layout and allows change settings on cluster-level and replica-level
+                        describes clusters layout and allows change settings on cluster-level and replica-level
                       # nullable: true
                       items:
                         type: object
@@ -147,25 +511,178 @@ spec:
                         properties:
                           name:
                             type: string
-                            description: "cluster name, used to identify set of ClickHouseKeeper servers and wide used during generate names of related Kubernetes resources"
+                            description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources"
                             minLength: 1
                             # See namePartClusterMaxLen const
                             maxLength: 15
                             pattern: "^[a-zA-Z0-9-]{0,15}$"
+                          settings:
+                            !!merge <<: *TypeSettings
+                            description: |
+                              optional, allows configure `clickhouse-server` settings inside <yandex>...</yandex> tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/`
+                              override top-level `chi.spec.configuration.settings`
+                              More details: https://clickhouse.tech/docs/en/operations/settings/settings/
+                          files:
+                            !!merge <<: *TypeFiles
+                            description: |
+                              optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`
+                              override top-level `chi.spec.configuration.files`
+                          templates:
+                            !!merge <<: *TypeTemplateNames
+                            description: |
+                              optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster
+                              override top-level `chi.spec.configuration.templates`
                           layout:
                             type: object
                             description: |
-                              describe current cluster layout, how many replicas
+                              describe current cluster layout, how much shards in cluster, how much replica in shard
+                              allows override settings on each shard and replica separatelly
                             # nullable: true
                             properties:
                               replicasCount:
                                 type: integer
-                                description: "how many replicas in ClickHouseKeeper cluster"
+                                description: |
+                                  how much replicas in each shards for current cluster will run in Kubernetes,
+                                  each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance,
+                                  every shard contains 1 replica by default"
+                              replicas:
+                                type: array
+                                description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do"
+                                # nullable: true
+                                items:
+                                  type: object
+                                  properties:
+                                    name:
+                                      type: string
+                                      description: "optional, by default replica name is generated, but you can override it and setup custom name"
+                                      minLength: 1
+                                      # See namePartShardMaxLen const
+                                      maxLength: 15
+                                      pattern: "^[a-zA-Z0-9-]{0,15}$"
+                                    settings:
+                                      !!merge <<: *TypeSettings
+                                      description: |
+                                        optional, allows configure `clickhouse-server` settings inside <yandex>...</yandex> tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/`
+                                        override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present
+                                        More details: https://clickhouse.tech/docs/en/operations/settings/settings/
+                                    files:
+                                      !!merge <<: *TypeFiles
+                                      description: |
+                                        optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`
+                                        override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents
+                                    templates:
+                                      !!merge <<: *TypeTemplateNames
+                                      description: |
+                                        optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica
+                                        override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`
+                                    shardsCount:
+                                      type: integer
+                                      description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`"
+                                      minimum: 1
+                                    shards:
+                                      type: array
+                                      description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents"
+                                      # nullable: true
+                                      items:
+                                        # Host
+                                        type: object
+                                        properties:
+                                          name:
+                                            type: string
+                                            description: "optional, by default shard name is generated, but you can override it and setup custom name"
+                                            minLength: 1
+                                            # See namePartReplicaMaxLen const
+                                            maxLength: 15
+                                            pattern: "^[a-zA-Z0-9-]{0,15}$"
+                                          zkPort:
+                                            type: integer
+                                            minimum: 1
+                                            maximum: 65535
+                                          raftPort:
+                                            type: integer
+                                            minimum: 1
+                                            maximum: 65535
+                                          settings:
+                                            !!merge <<: *TypeSettings
+                                            description: |
+                                              optional, allows configure `clickhouse-server` settings inside <yandex>...</yandex> tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/`
+                                              override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings`
+                                              More details: https://clickhouse.tech/docs/en/operations/settings/settings/
+                                          files:
+                                            !!merge <<: *TypeFiles
+                                            description: |
+                                              optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`
+                                              override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents
+                                          templates:
+                                            !!merge <<: *TypeTemplateNames
+                                            description: |
+                                              optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica
+                                              override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates`
                 templates:
                   type: object
                   description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it"
                   # nullable: true
                   properties:
+                    hostTemplates:
+                      type: array
+                      description: "hostTemplate will use during apply to generate `clickhose-server` config files"
+                      # nullable: true
+                      items:
+                        type: object
+                        #required:
+                        #  - name
+                        properties:
+                          name:
+                            description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`"
+                            type: string
+                          portDistribution:
+                            type: array
+                            description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs"
+                            # nullable: true
+                            items:
+                              type: object
+                              #required:
+                              #  - type
+                              properties:
+                                type:
+                                  type: string
+                                  description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network"
+                                  enum:
+                                    # List PortDistributionXXX constants
+                                    - ""
+                                    - "Unspecified"
+                                    - "ClusterScopeIndex"
+                          spec:
+                            # Host
+                            type: object
+                            properties:
+                              name:
+                                type: string
+                                description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`"
+                                minLength: 1
+                                # See namePartReplicaMaxLen const
+                                maxLength: 15
+                                pattern: "^[a-zA-Z0-9-]{0,15}$"
+                              zkPort:
+                                type: integer
+                                minimum: 1
+                                maximum: 65535
+                              raftPort:
+                                type: integer
+                                minimum: 1
+                                maximum: 65535
+                              settings:
+                                !!merge <<: *TypeSettings
+                                description: |
+                                  optional, allows configure `clickhouse-server` settings inside <yandex>...</yandex> tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/`
+                                  More details: https://clickhouse.tech/docs/en/operations/settings/settings/
+                              files:
+                                !!merge <<: *TypeFiles
+                                description: |
+                                  optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`
+                              templates:
+                                !!merge <<: *TypeTemplateNames
+                                description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do"
                     podTemplates:
                       type: array
                       description: |
@@ -180,6 +697,83 @@ spec:
                           name:
                             type: string
                             description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`"
+                          generateName:
+                            type: string
+                            description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables"
+                          zone:
+                            type: object
+                            description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`"
+                            #required:
+                            #  - values
+                            properties:
+                              key:
+                                type: string
+                                description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`"
+                              values:
+                                type: array
+                                description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`"
+                                # nullable: true
+                                items:
+                                  type: string
+                          distribution:
+                            type: string
+                            description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`"
+                            enum:
+                              - ""
+                              - "Unspecified"
+                              - "OnePerHost"
+                          podDistribution:
+                            type: array
+                            description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster"
+                            # nullable: true
+                            items:
+                              type: object
+                              #required:
+                              #  - type
+                              properties:
+                                type:
+                                  type: string
+                                  description: "you can define multiple affinity policy types"
+                                  enum:
+                                    # List PodDistributionXXX constants
+                                    - ""
+                                    - "Unspecified"
+                                    - "ClickHouseAntiAffinity"
+                                    - "ShardAntiAffinity"
+                                    - "ReplicaAntiAffinity"
+                                    - "AnotherNamespaceAntiAffinity"
+                                    - "AnotherClickHouseInstallationAntiAffinity"
+                                    - "AnotherClusterAntiAffinity"
+                                    - "MaxNumberPerNode"
+                                    - "NamespaceAffinity"
+                                    - "ClickHouseInstallationAffinity"
+                                    - "ClusterAffinity"
+                                    - "ShardAffinity"
+                                    - "ReplicaAffinity"
+                                    - "PreviousTailAffinity"
+                                    - "CircularReplication"
+                                scope:
+                                  type: string
+                                  description: "scope for apply each podDistribution"
+                                  enum:
+                                    # list PodDistributionScopeXXX constants
+                                    - ""
+                                    - "Unspecified"
+                                    - "Shard"
+                                    - "Replica"
+                                    - "Cluster"
+                                    - "ClickHouseInstallation"
+                                    - "Namespace"
+                                number:
+                                  type: integer
+                                  description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type"
+                                  minimum: 0
+                                  maximum: 65535
+                                topologyKey:
+                                  type: string
+                                  description: |
+                                    use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`,
+                                    more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity"
                           metadata:
                             type: object
                             description: |
@@ -195,7 +789,8 @@ spec:
                             x-kubernetes-preserve-unknown-fields: true
                     volumeClaimTemplates:
                       type: array
-                      description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else"
+                      description: |
+                        allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else
                       # nullable: true
                       items:
                         type: object
@@ -211,6 +806,8 @@ spec:
                               cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`,
                               shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate`
                               replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate`
+                          provisioner: *TypePVCProvisioner
+                          reclaimPolicy: *TypePVCReclaimPolicy
                           metadata:
                             type: object
                             description: |
@@ -244,6 +841,12 @@ spec:
                               cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate`
                               shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate`
                               replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate`
+                          generateName:
+                            type: string
+                            description: |
+                              allows define format for generated `Service` name,
+                              look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates
+                              for details about available template variables"
                           metadata:
                             # TODO specify ObjectMeta
                             type: object

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml

@@ -7,7 +7,7 @@ kind: CustomResourceDefinition
 metadata:
   name: clickhouseoperatorconfigurations.clickhouse.altinity.com
   labels:
-    clickhouse.altinity.com/chop: 0.23.4
+    clickhouse.altinity.com/chop: 0.25.2
 spec:
   group: clickhouse.altinity.com
   scope: Namespaced
@@ -137,6 +137,7 @@ spec:
                                 items:
                                   type: object
                                   description: "setting: value pairs for configuration restart policy"
+                                  x-kubernetes-preserve-unknown-fields: true
                     access:
                       type: object
                       description: "parameters which use for connect to clickhouse from clickhouse-operator deployment"
@@ -181,6 +182,47 @@ spec:
                               minimum: 1
                               maximum: 600
                               description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds."
+                    addons:
+                      type: object
+                      description: "Configuration addons specifies additional settings"
+                      properties:
+                        rules:
+                          type: array
+                          description: "Array of set of rules per specified ClickHouse versions"
+                          items:
+                            type: object
+                            properties:
+                              version:
+                                type: string
+                                description: "ClickHouse version expression"
+                              spec:
+                                type: object
+                                description: "spec"
+                                properties:
+                                  configuration:
+                                    type: object
+                                    description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource"
+                                    properties:
+                                      users:
+                                        type: object
+                                        description: "see same section from CR spec"
+                                        x-kubernetes-preserve-unknown-fields: true
+                                      profiles:
+                                        type: object
+                                        description: "see same section from CR spec"
+                                        x-kubernetes-preserve-unknown-fields: true
+                                      quotas:
+                                        type: object
+                                        description: "see same section from CR spec"
+                                        x-kubernetes-preserve-unknown-fields: true
+                                      settings:
+                                        type: object
+                                        description: "see same section from CR spec"
+                                        x-kubernetes-preserve-unknown-fields: true
+                                      files:
+                                        type: object
+                                        description: "see same section from CR spec"
+                                        x-kubernetes-preserve-unknown-fields: true
                     metrics:
                       type: object
                       description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator"
@@ -323,6 +365,19 @@ spec:
                             include:
                               !!merge <<: *TypeStringBool
                               description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster"
+                            replicas:
+                              type: object
+                              description: "Whether the operator during reconcile procedure should wait for replicas to catch-up"
+                              properties:
+                                all:
+                                  !!merge <<: *TypeStringBool
+                                  description: "Whether the operator during reconcile procedure should wait for all replicas to catch-up"
+                                new:
+                                  !!merge <<: *TypeStringBool
+                                  description: "Whether the operator during reconcile procedure should wait for new replicas to catch-up"
+                                delay:
+                                  type: integer
+                                  description: "replication max absolute delay to consider replica is not delayed"
                 annotation:
                   type: object
                   description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources"
@@ -373,6 +428,40 @@ spec:
                         - "LabelClusterScopeCycleSize"
                         - "LabelClusterScopeCycleIndex"
                         - "LabelClusterScopeCycleOffset"
+                metrics:
+                  type: object
+                  description: "defines metrics exporter options"
+                  properties:
+                    labels:
+                      type: object
+                      description: "defines metric labels options"
+                      properties:
+                        exclude:
+                          type: array
+                          description: |
+                            When adding labels to a metric exclude labels with names from the following list
+                          items:
+                            type: string
+                status:
+                  type: object
+                  description: "defines status options"
+                  properties:
+                    fields:
+                      type: object
+                      description: "defines status fields options"
+                      properties:
+                        action:
+                          !!merge <<: *TypeStringBool
+                          description: "Whether the operator should fill status field 'action'"
+                        actions:
+                          !!merge <<: *TypeStringBool
+                          description: "Whether the operator should fill status field 'actions'"
+                        error:
+                          !!merge <<: *TypeStringBool
+                          description: "Whether the operator should fill status field 'error'"
+                        errors:
+                          !!merge <<: *TypeStringBool
+                          description: "Whether the operator should fill status field 'errors'"
                 statefulSet:
                   type: object
                   description: "define StatefulSet-specific parameters"

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/ClickHouse_Queries_dashboard.json

@@ -147,8 +147,8 @@
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
-          "query": "SELECT\r\n    t,\r\n    arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n    SELECT t, groupArray((q, c)) AS groupArr\r\n    FROM (\r\n        SELECT\r\n            (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n            normalizeQuery(query) AS q,\r\n            count() c\r\n        FROM cluster('all-sharded',system.query_log)\r\n        WHERE $timeFilter\r\n        AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n        $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n        $conditionalTest(AND initial_user IN ($user), $user)\r\n        $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n        $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n        AND normalized_query_hash GLOBAL IN (\r\n            SELECT normalized_query_hash AS h\r\n            FROM cluster('all-sharded',system.query_log)\r\n            WHERE $timeFilter\r\n                AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n                $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n                $conditionalTest(AND type IN ($type), $type)\r\n                $conditionalTest(AND initial_user IN ($user), $user)\r\n                $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n                $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n            GROUP BY h\r\n            ORDER BY count() DESC\r\n            LIMIT $top\r\n            SETTINGS skip_unavailable_shards=1\r\n        )\r\n        GROUP BY t, query\r\n        ORDER BY t\r\n    )\r\n    GROUP BY t\r\n    ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1",
-          "rawQuery": "SELECT\r\n    t,\r\n    arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n    SELECT t, groupArray((q, c)) AS groupArr\r\n    FROM (\r\n        SELECT\r\n            (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n            normalizeQuery(query) AS q,\r\n            count() c\r\n        FROM cluster('all-sharded',system.query_log)\r\n        WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n        AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n         \r\n         \r\n         \r\n         \r\n        AND normalized_query_hash GLOBAL IN (\r\n            SELECT normalized_query_hash AS h\r\n            FROM cluster('all-sharded',system.query_log)\r\n            WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n                AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n                 \r\n                 \r\n                 \r\n                 \r\n                 \r\n            GROUP BY h\r\n            ORDER BY count() DESC\r\n            LIMIT 30\r\n            SETTINGS skip_unavailable_shards=1\r\n        )\r\n        GROUP BY t, query\r\n        ORDER BY t\r\n    )\r\n    GROUP BY t\r\n    ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1",
+          "query": "SELECT\r\n    t,\r\n    arrayMap(a -> (a.1, a.2 / (t/1000 - lagInFrame(t/1000,1,0) OVER ()) ), groupArr)\r\nFROM (\r\n    SELECT t, groupArray((q, c)) AS groupArr\r\n    FROM (\r\n        SELECT\r\n            (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n            normalizeQuery(query) AS q,\r\n            count() c\r\n        FROM cluster('all-sharded',system.query_log)\r\n        WHERE $timeFilter\r\n        AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n        $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n        $conditionalTest(AND initial_user IN ($user), $user)\r\n        $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n        $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n        AND normalized_query_hash GLOBAL IN (\r\n            SELECT normalized_query_hash AS h\r\n            FROM cluster('all-sharded',system.query_log)\r\n            WHERE $timeFilter\r\n                AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n                $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n                $conditionalTest(AND type IN ($type), $type)\r\n                $conditionalTest(AND initial_user IN ($user), $user)\r\n                $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n                $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n            GROUP BY h\r\n            ORDER BY count() DESC\r\n            LIMIT $top\r\n            SETTINGS skip_unavailable_shards=1\r\n        )\r\n        GROUP BY t, query\r\n        ORDER BY t\r\n    )\r\n    GROUP BY t\r\n    ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1",
+          "rawQuery": "SELECT\r\n    t,\r\n    arrayMap(a -> (a.1, a.2 / (t/1000 - lagInFrame(t/1000,1,0) OVER ()) ), groupArr)\r\nFROM (\r\n    SELECT t, groupArray((q, c)) AS groupArr\r\n    FROM (\r\n        SELECT\r\n            (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n            normalizeQuery(query) AS q,\r\n            count() c\r\n        FROM cluster('all-sharded',system.query_log)\r\n        WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n        AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n         \r\n         \r\n         \r\n         \r\n        AND normalized_query_hash GLOBAL IN (\r\n            SELECT normalized_query_hash AS h\r\n            FROM cluster('all-sharded',system.query_log)\r\n            WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n                AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n                 \r\n                 \r\n                 \r\n                 \r\n                 \r\n            GROUP BY h\r\n            ORDER BY count() DESC\r\n            LIMIT 30\r\n            SETTINGS skip_unavailable_shards=1\r\n        )\r\n        GROUP BY t, query\r\n        ORDER BY t\r\n    )\r\n    GROUP BY t\r\n    ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1",
           "refId": "A",
           "resultFormat": "time_series",
           "round": "0s",
@@ -743,7 +743,7 @@
           "interval": "",
           "intervalFactor": 2,
           "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n    AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n    $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n    $conditionalTest(AND initial_user IN ($user), $user)\n    $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n    $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\n",
-          "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n    AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n     \n     \n     \n      GROUP BY t ORDER BY t)",
+          "rawQuery": "SELECT t, c/(t/1000 - lagInFrame(t/1000,1,0) OVER ()) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n    AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n     \n     \n     \n      GROUP BY t ORDER BY t)",
           "refId": "A",
           "resultFormat": "time_series",
           "round": "0s",

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/_helpers.tpl

@@ -1,4 +1,15 @@
 {{/* vim: set filetype=go-template: */}}
+{{/*
+Allow the release namespace to be overridden for multi-namespace deployments in combined charts
+*/}}
+{{- define "altinity-clickhouse-operator.namespace" -}}
+  {{- if .Values.namespaceOverride -}}
+    {{- .Values.namespaceOverride -}}
+  {{- else -}}
+    {{- .Release.Namespace -}}
+  {{- end -}}
+{{- end -}}
+
 {{/*
 Expand the name of the chart.
 */}}
@@ -40,8 +51,8 @@ helm.sh/chart: {{ include "altinity-clickhouse-operator.chart" . }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
-{{- if .Values.podLabels }}
-{{ toYaml .Values.podLabels }}
+{{- if .Values.commonLabels }}
+{{ toYaml .Values.commonLabels }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
@@ -54,6 +65,17 @@ app.kubernetes.io/name: {{ include "altinity-clickhouse-operator.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end -}}
 
+{{/*
+Common annotations
+*/}}
+{{- define "altinity-clickhouse-operator.annotations" -}}
+meta.helm.sh/release-name: {{ .Release.Name }}
+meta.helm.sh/release-namespace: {{ .Release.Namespace }}
+{{- if .Values.commonAnnotations }}
+{{ toYaml .Values.commonAnnotations }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Create the name of the service account to use
 */}}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-configmap.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.dashboards.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "altinity-clickhouse-operator.fullname" . }}-dashboards
+  namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
+  labels:
+    {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+    {{- if .Values.dashboards.additionalLabels }}
+    {{- toYaml .Values.dashboards.additionalLabels | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- include "altinity-clickhouse-operator.annotations" . | nindent 4 }}
+    {{- if .Values.dashboards.annotations }}
+    {{- toYaml .Values.dashboards.annotations | nindent 4 }}
+    {{- end }}
+data:
+{{- range $path, $_ := .Files.Glob "files/*.json" }}
+  {{ $path | trimPrefix "files/" }}: |- {{ $.Files.Get $path | nindent 4 -}}
+{{ end }}
+{{- end }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-secret.yaml

@@ -1,21 +0,0 @@
-{{- if .Values.dashboards.enabled }}
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ include "altinity-clickhouse-operator.fullname" . }}-dashboards
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }}
-{{- if .Values.dashboards.additionalLabels }}
-    {{- toYaml .Values.dashboards.additionalLabels | nindent 4 }}
-{{- end }}
-{{- with .Values.dashboards.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-{{- end }}
-type: Opaque
-data:
-{{- range $path, $_ := .Files.Glob "files/*.json" }}
-  {{ $path | trimPrefix "files/" }}: {{ $.Files.Get $path | b64enc -}}
-{{ end }}
-{{- end }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.rbac.create -}}
+{{- if (and .Values.rbac.create (not .Values.rbac.namespaceScoped)) -}}
 # Specifies either
 #   ClusterRole
 # or
@@ -12,7 +12,7 @@ metadata:
   name: {{ include "altinity-clickhouse-operator.fullname" . }}
   #namespace: kube-system
   labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
-  namespace: {{ .Release.Namespace }}
+  annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }}
 rules:
   #
   # Core API group

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.rbac.create -}}
+{{- if (and .Values.rbac.create (not .Values.rbac.namespaceScoped)) -}}
 # Specifies either
 #   ClusterRoleBinding between ClusterRole and ServiceAccount.
 # or
@@ -11,7 +11,7 @@ metadata:
   name: {{ include "altinity-clickhouse-operator.fullname" . }}
   #namespace: kube-system
   labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
-  namespace: {{ .Release.Namespace }}
+  annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
@@ -19,5 +19,15 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: {{ include "altinity-clickhouse-operator.serviceAccountName" . }}
-    namespace: {{ .Release.Namespace }}
+    namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
+
+# Template Parameters:
+#
+# NAMESPACE=kube-system
+# COMMENT=
+# ROLE_KIND=Role
+# ROLE_NAME=clickhouse-operator
+# ROLE_BINDING_KIND=RoleBinding
+# ROLE_BINDING_NAME=clickhouse-operator
+#
 {{- end }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml

@@ -8,6 +8,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ printf "%s-confd-files" (include "altinity-clickhouse-operator.fullname" .) }}
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
   labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+  annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }}
 data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.confdFiles) | nindent 2 }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml

@@ -8,6 +8,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ printf "%s-configd-files" (include "altinity-clickhouse-operator.fullname" .) }}
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
   labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+  annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }}
 data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.configdFiles) | nindent 2 }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml

@@ -8,6 +8,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ printf "%s-files" (include "altinity-clickhouse-operator.fullname" .) }}
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
   labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+  annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }}
 data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.files) | nindent 2 }}