Add clickhouse-operator

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

packages/core/platform/bundles/distro-full.yaml

@@ -76,6 +76,12 @@ releases:
   namespace: cozy-kafka-operator
   dependsOn: [cilium,kubeovn]
 
+- name: clickhouse-operator
+  releaseName: clickhouse-operator
+  chart: cozy-clickhouse-operator
+  namespace: cozy-clickhouse-operator
+  dependsOn: [cilium,kubeovn]
+
 - name: rabbitmq-operator
   releaseName: rabbitmq-operator
   chart: cozy-rabbitmq-operator

packages/core/platform/bundles/distro-hosted.yaml

@@ -50,6 +50,12 @@ releases:
   namespace: cozy-kafka-operator
   dependsOn: [cilium,kubeovn]
 
+- name: clickhouse-operator
+  releaseName: clickhouse-operator
+  chart: cozy-clickhouse-operator
+  namespace: cozy-clickhouse-operator
+  dependsOn: [cilium,kubeovn]
+
 - name: rabbitmq-operator
   releaseName: rabbitmq-operator
   chart: cozy-rabbitmq-operator

packages/core/platform/bundles/paas-full.yaml

@@ -105,6 +105,12 @@ releases:
   namespace: cozy-kafka-operator
   dependsOn: [cilium,kubeovn]
 
+- name: clickhouse-operator
+  releaseName: clickhouse-operator
+  chart: cozy-clickhouse-operator
+  namespace: cozy-clickhouse-operator
+  dependsOn: [cilium,kubeovn]
+
 - name: rabbitmq-operator
   releaseName: rabbitmq-operator
   chart: cozy-rabbitmq-operator

packages/core/platform/bundles/paas-hosted.yaml

@@ -50,6 +50,12 @@ releases:
   namespace: cozy-kafka-operator
   dependsOn: [cilium,kubeovn]
 
+- name: clickhouse-operator
+  releaseName: clickhouse-operator
+  chart: cozy-clickhouse-operator
+  namespace: cozy-clickhouse-operator
+  dependsOn: [cilium,kubeovn]
+
 - name: rabbitmq-operator
   releaseName: rabbitmq-operator
   chart: cozy-rabbitmq-operator

packages/system/clickhouse-operator/Chart.yaml

@@ -0,0 +1,2 @@
+name: cozy-clickhouse-operator
+version: 0.3.0

packages/system/clickhouse-operator/Makefile

@@ -0,0 +1,17 @@
+NAME=clickhouse-operator
+NAMESPACE=cozy-clickhouse-operator
+
+show:
+	helm template --dry-run=server -n $(NAMESPACE) $(NAME) .
+
+apply:
+	helm upgrade -i -n $(NAMESPACE) $(NAME) .
+
+diff:
+	helm diff upgrade --allow-unreleased --normalize-manifests -n $(NAMESPACE) $(NAME) .
+
+update:
+	rm -rf charts
+	helm repo add clickhouse-operator https://docs.altinity.com/clickhouse-operator/
+	helm repo update clickhouse-operator
+	helm pull clickhouse-operator/altinity-clickhouse-operator --untar --untardir charts

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/Chart.yaml

@@ -0,0 +1,17 @@
+apiVersion: v2
+appVersion: 0.23.4
+description: 'Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator).  The
+  ClickHouse Operator creates, configures and manages ClickHouse clusters running
+  on Kubernetes.  For upgrade please install CRDs separately: ```bash   kubectl apply
+  -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml   kubectl
+  apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml   kubectl
+  apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml
+  ```'
+home: https://github.com/Altinity/clickhouse-operator
+icon: https://logosandtypes.com/wp-content/uploads/2020/12/altinity.svg
+maintainers:
+- email: support@altinity.com
+  name: altinity
+name: altinity-clickhouse-operator
+type: application
+version: 0.23.4

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md

@@ -0,0 +1,65 @@
+# altinity-clickhouse-operator
+
+![Version: 0.23.4](https://img.shields.io/badge/Version-0.23.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.23.4](https://img.shields.io/badge/AppVersion-0.23.4-informational?style=flat-square)
+
+Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator).
+
+The ClickHouse Operator creates, configures and manages ClickHouse clusters running on Kubernetes.
+
+For upgrade please install CRDs separately:
+```bash
+  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml
+  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml
+  kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml
+```
+
+**Homepage:** <https://github.com/Altinity/clickhouse-operator>
+
+## Maintainers
+
+| Name | Email | Url |
+| ---- | ------ | --- |
+| altinity | <support@altinity.com> |  |
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| additionalResources | list | `[]` | list of additional resources to create (are processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator, look `kubectl explain chi` for details |
+| affinity | object | `{}` | affinity for scheduler pod assignment, look `kubectl explain pod.spec.affinity` for details |
+| configs | object | check the values.yaml file for the config content, auto-generated from latest operator release | clickhouse-operator configs |
+| dashboards.additionalLabels | object | `{"grafana_dashboard":""}` | labels to add to a secret with dashboards |
+| dashboards.annotations | object | `{}` | annotations to add to a secret with dashboards |
+| dashboards.enabled | bool | `false` | provision grafana dashboards as secrets (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-6.33.1/charts/grafana/values.yaml#L679 ) |
+| dashboards.grafana_folder | string | `"clickhouse"` |  |
+| fullnameOverride | string | `""` | full name of the chart. |
+| imagePullSecrets | list | `[]` | image pull secret for private images in clickhouse-operator pod  possible value format [{"name":"your-secret-name"}]  look `kubectl explain pod.spec.imagePullSecrets` for details |
+| metrics.containerSecurityContext | object | `{}` |  |
+| metrics.enabled | bool | `true` |  |
+| metrics.env | list | `[]` | additional environment variables for the deployment of metrics-exporter containers possible format value [{"name": "SAMPLE", "value": "text"}] |
+| metrics.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| metrics.image.repository | string | `"altinity/metrics-exporter"` | image repository |
+| metrics.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) |
+| metrics.resources | object | `{}` | custom resource configuration |
+| nameOverride | string | `""` | override name of the chart |
+| nodeSelector | object | `{}` | node for scheduler pod assignment, look `kubectl explain pod.spec.nodeSelector` for details |
+| operator.containerSecurityContext | object | `{}` |  |
+| operator.env | list | `[]` | additional environment variables for the clickhouse-operator container in deployment possible format value [{"name": "SAMPLE", "value": "text"}] |
+| operator.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| operator.image.repository | string | `"altinity/clickhouse-operator"` | image repository |
+| operator.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) |
+| operator.resources | object | `{}` | custom resource configuration, look `kubectl explain pod.spec.containers.resources` for details |
+| podAnnotations | object | `{"clickhouse-operator-metrics/port":"9999","clickhouse-operator-metrics/scrape":"true","prometheus.io/port":"8888","prometheus.io/scrape":"true"}` | annotations to add to the clickhouse-operator pod, look `kubectl explain pod.spec.annotations` for details |
+| podLabels | object | `{}` | labels to add to the clickhouse-operator pod |
+| podSecurityContext | object | `{}` |  |
+| rbac.create | bool | `true` | specifies whether cluster roles and cluster role bindings should be created |
+| secret.create | bool | `true` | create a secret with operator credentials |
+| secret.password | string | `"clickhouse_operator_password"` | operator credentials password |
+| secret.username | string | `"clickhouse_operator"` | operator credentials username |
+| serviceAccount.annotations | object | `{}` | annotations to add to the service account |
+| serviceAccount.create | bool | `true` | specifies whether a service account should be created |
+| serviceAccount.name | string | `nil` | the name of the service account to use; if not set and create is true, a name is generated using the fullname template |
+| serviceMonitor.additionalLabels | object | `{}` | additional labels for service monitor |
+| serviceMonitor.enabled | bool | `false` | ServiceMonitor Custom resource is created for a (prometheus-operator)[https://github.com/prometheus-operator/prometheus-operator] |
+| tolerations | list | `[]` | tolerations for scheduler pod assignment, look `kubectl explain pod.spec.tolerations` for details |
+

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml

@@ -0,0 +1,263 @@
+# Template Parameters:
+#
+# OPERATOR_VERSION=0.23.4
+#
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com
+  labels:
+    clickhouse-keeper.altinity.com/chop: 0.23.4
+spec:
+  group: clickhouse-keeper.altinity.com
+  scope: Namespaced
+  names:
+    kind: ClickHouseKeeperInstallation
+    singular: clickhousekeeperinstallation
+    plural: clickhousekeeperinstallations
+    shortNames:
+      - chk
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      additionalPrinterColumns:
+        - name: status
+          type: string
+          description: CHK status
+          jsonPath: .status.status
+        - name: replicas
+          type: integer
+          description: Replica count
+          priority: 1 # show in wide view
+          jsonPath: .status.replicas
+        - name: age
+          type: date
+          description: Age of the resource
+          # Displayed in all priorities
+          jsonPath: .metadata.creationTimestamp
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - spec
+          description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one ClickHouse Keeper cluster"
+          properties:
+            apiVersion:
+              type: string
+              description: |
+                APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            kind:
+              type: string
+              description: |
+                Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            metadata:
+              type: object
+            status:
+              type: object
+              description: |
+                Current ClickHouseKeeperInstallation status, contains many fields like overall status, desired replicas and ready replica list with their endpoints
+              properties:
+                chop-version:
+                  type: string
+                  description: "ClickHouse operator version"
+                chop-commit:
+                  type: string
+                  description: "ClickHouse operator git commit SHA"
+                chop-date:
+                  type: string
+                  description: "ClickHouse operator build date"
+                chop-ip:
+                  type: string
+                  description: "IP address of the operator's pod which managed this CHI"
+                status:
+                  type: string
+                  description: "Status"
+                replicas:
+                  type: integer
+                  format: int32
+                  description: Replicas is the number of number of desired replicas in the cluster
+                readyReplicas:
+                  type: array
+                  description: ReadyReplicas is the array of endpoints of those ready replicas in the cluster
+                  items:
+                    type: object
+                    properties:
+                      host:
+                        type: string
+                        description: dns name or ip address for Keeper node
+                      port:
+                        type: integer
+                        minimum: 0
+                        maximum: 65535
+                        description: TCP port which used to connect to Keeper node
+                      secure:
+                        type: string
+                        description: if a secure connection to Keeper is required
+                normalized:
+                  type: object
+                  description: "Normalized CHK requested"
+                  x-kubernetes-preserve-unknown-fields: true
+                normalizedCompleted:
+                  type: object
+                  description: "Normalized CHK completed"
+                  x-kubernetes-preserve-unknown-fields: true
+            spec:
+              type: object
+              description: KeeperSpec defines the desired state of a Keeper cluster
+              properties:
+                namespaceDomainPattern:
+                  type: string
+                  description: |
+                    Custom domain pattern which will be used for DNS names of `Service` or `Pod`.
+                    Typical use scenario - custom cluster domain in Kubernetes cluster
+                    Example: %s.svc.my.test
+                replicas:
+                  type: integer
+                  format: int32
+                  description: |
+                    Replicas is the expected size of the keeper cluster.
+                    The valid range of size is from 1 to 7.
+                  minimum: 1
+                  maximum: 7
+                configuration:
+                  type: object
+                  description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource"
+                  # nullable: true
+                  properties:
+                    settings:
+                      type: object
+                      description: "allows configure multiple aspects and behavior for `clickhouse-keeper` instance"
+                      x-kubernetes-preserve-unknown-fields: true
+                    clusters:
+                      type: array
+                      description: |
+                        describes ClickHouseKeeper clusters layout and allows change settings on cluster-level and replica-level
+                      # nullable: true
+                      items:
+                        type: object
+                        #required:
+                        #  - name
+                        properties:
+                          name:
+                            type: string
+                            description: "cluster name, used to identify set of ClickHouseKeeper servers and wide used during generate names of related Kubernetes resources"
+                            minLength: 1
+                            # See namePartClusterMaxLen const
+                            maxLength: 15
+                            pattern: "^[a-zA-Z0-9-]{0,15}$"
+                          layout:
+                            type: object
+                            description: |
+                              describe current cluster layout, how many replicas
+                            # nullable: true
+                            properties:
+                              replicasCount:
+                                type: integer
+                                description: "how many replicas in ClickHouseKeeper cluster"
+                templates:
+                  type: object
+                  description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it"
+                  # nullable: true
+                  properties:
+                    podTemplates:
+                      type: array
+                      description: |
+                        podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone
+                        More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates
+                      # nullable: true
+                      items:
+                        type: object
+                        #required:
+                        #  - name
+                        properties:
+                          name:
+                            type: string
+                            description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`"
+                          metadata:
+                            type: object
+                            description: |
+                              allows pass standard object's metadata from template to Pod
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+                          spec:
+                            # TODO specify PodSpec
+                            type: object
+                            description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details"
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+                    volumeClaimTemplates:
+                      type: array
+                      description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else"
+                      # nullable: true
+                      items:
+                        type: object
+                        #required:
+                        #  - name
+                        #  - spec
+                        properties:
+                          name:
+                            type: string
+                            description: |
+                              template name, could use to link inside
+                              top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`,
+                              cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`,
+                              shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate`
+                              replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate`
+                          metadata:
+                            type: object
+                            description: |
+                              allows to pass standard object's metadata from template to PVC
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+                          spec:
+                            type: object
+                            description: |
+                              allows define all aspects of `PVC` resource
+                              More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+                    serviceTemplates:
+                      type: array
+                      description: |
+                        allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level
+                      # nullable: true
+                      items:
+                        type: object
+                        #required:
+                        #  - name
+                        #  - spec
+                        properties:
+                          name:
+                            type: string
+                            description: |
+                              template name, could use to link inside
+                              chi-level `chi.spec.defaults.templates.serviceTemplate`
+                              cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate`
+                              shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate`
+                              replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate`
+                          metadata:
+                            # TODO specify ObjectMeta
+                            type: object
+                            description: |
+                              allows pass standard object's metadata from template to Service
+                              Could be use for define specificly for Cloud Provider metadata which impact to behavior of service
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+                          spec:
+                            # TODO specify ServiceSpec
+                            type: object
+                            description: |
+                              describe behavior of generated Service
+                              More info: https://kubernetes.io/docs/concepts/services-networking/service/
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml

@@ -0,0 +1,415 @@
+# Template Parameters:
+#
+# NONE
+#
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: clickhouseoperatorconfigurations.clickhouse.altinity.com
+  labels:
+    clickhouse.altinity.com/chop: 0.23.4
+spec:
+  group: clickhouse.altinity.com
+  scope: Namespaced
+  names:
+    kind: ClickHouseOperatorConfiguration
+    singular: clickhouseoperatorconfiguration
+    plural: clickhouseoperatorconfigurations
+    shortNames:
+      - chopconf
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      additionalPrinterColumns:
+        - name: namespaces
+          type: string
+          description: Watch namespaces
+          jsonPath: .status
+        - name: age
+          type: date
+          description: Age of the resource
+          # Displayed in all priorities
+          jsonPath: .metadata.creationTimestamp
+      schema:
+        openAPIV3Schema:
+          type: object
+          description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md"
+          x-kubernetes-preserve-unknown-fields: true
+          properties:
+            status:
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+            spec:
+              type: object
+              description: |
+                Allows to define settings of the clickhouse-operator.
+                More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml
+                Check into etc-clickhouse-operator* ConfigMaps if you need more control
+              x-kubernetes-preserve-unknown-fields: true
+              properties:
+                watch:
+                  type: object
+                  description: "Parameters for watch kubernetes resources which used by clickhouse-operator deployment"
+                  properties:
+                    namespaces:
+                      type: array
+                      description: "List of namespaces where clickhouse-operator watches for events."
+                      items:
+                        type: string
+                clickhouse:
+                  type: object
+                  description: "Clickhouse related parameters used by clickhouse-operator"
+                  properties:
+                    configuration:
+                      type: object
+                      properties:
+                        file:
+                          type: object
+                          properties:
+                            path:
+                              type: object
+                              description: |
+                                Each 'path' can be either absolute or relative.
+                                In case path is absolute - it is used as is.
+                                In case path is relative - it is relative to the folder where configuration file you are reading right now is located.
+                              properties:
+                                common:
+                                  type: string
+                                  description: |
+                                    Path to the folder where ClickHouse configuration files common for all instances within a CHI are located.
+                                    Default value - config.d
+                                host:
+                                  type: string
+                                  description: |
+                                    Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located.
+                                    Default value - conf.d
+                                user:
+                                  type: string
+                                  description: |
+                                    Path to the folder where ClickHouse configuration files with users settings are located.
+                                    Files are common for all instances within a CHI.
+                                    Default value - users.d
+                        user:
+                          type: object
+                          description: "Default parameters for any user which will create"
+                          properties:
+                            default:
+                              type: object
+                              properties:
+                                profile:
+                                  type: string
+                                  description: "ClickHouse server configuration `<profile>...</profile>` for any <user>"
+                                quota:
+                                  type: string
+                                  description: "ClickHouse server configuration `<quota>...</quota>` for any <user>"
+                                networksIP:
+                                  type: array
+                                  description: "ClickHouse server configuration `<networks><ip>...</ip></networks>` for any <user>"
+                                  items:
+                                    type: string
+                                password:
+                                  type: string
+                                  description: "ClickHouse server configuration `<password>...</password>` for any <user>"
+                        network:
+                          type: object
+                          description: "Default network parameters for any user which will create"
+                          properties:
+                            hostRegexpTemplate:
+                              type: string
+                              description: "ClickHouse server configuration `<host_regexp>...</host_regexp>` for any <user>"
+                    configurationRestartPolicy:
+                      type: object
+                      description: "Configuration restart policy describes what configuration changes require ClickHouse restart"
+                      properties:
+                        rules:
+                          type: array
+                          description: "Array of set of rules per specified ClickHouse versions"
+                          items:
+                            type: object
+                            properties:
+                              version:
+                                type: string
+                                description: "ClickHouse version expression"
+                              rules:
+                                type: array
+                                description: "Set of configuration rules for specified ClickHouse version"
+                                items:
+                                  type: object
+                                  description: "setting: value pairs for configuration restart policy"
+                    access:
+                      type: object
+                      description: "parameters which use for connect to clickhouse from clickhouse-operator deployment"
+                      properties:
+                        scheme:
+                          type: string
+                          description: "The scheme to user for connecting to ClickHouse. Possible values: http, https, auto"
+                        username:
+                          type: string
+                          description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName"
+                        password:
+                          type: string
+                          description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName"
+                        rootCA:
+                          type: string
+                          description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse"
+                        secret:
+                          type: object
+                          properties:
+                            namespace:
+                              type: string
+                              description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances"
+                            name:
+                              type: string
+                              description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances"
+                        port:
+                          type: integer
+                          minimum: 1
+                          maximum: 65535
+                          description: "Port to be used by operator to connect to ClickHouse instances"
+                        timeouts:
+                          type: object
+                          description: "Timeouts used to limit connection and queries from the operator to ClickHouse instances, In seconds"
+                          properties:
+                            connect:
+                              type: integer
+                              minimum: 1
+                              maximum: 10
+                              description: "Timout to setup connection from the operator to ClickHouse instances. In seconds."
+                            query:
+                              type: integer
+                              minimum: 1
+                              maximum: 600
+                              description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds."
+                    metrics:
+                      type: object
+                      description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator"
+                      properties:
+                        timeouts:
+                          type: object
+                          description: |
+                            Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances
+                            Specified in seconds.
+                          properties:
+                            collect:
+                              type: integer
+                              minimum: 1
+                              maximum: 600
+                              description: |
+                                Timeout used to limit metrics collection request. In seconds.
+                                Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle.
+                                All collected metrics are returned.
+                template:
+                  type: object
+                  description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment"
+                  properties:
+                    chi:
+                      type: object
+                      properties:
+                        policy:
+                          type: string
+                          description: |
+                            CHI template updates handling policy
+                            Possible policy values:
+                              - ReadOnStart. Accept CHIT updates on the operators start only.
+                              - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI
+                          enum:
+                            - ""
+                            - "ReadOnStart"
+                            - "ApplyOnNextReconcile"
+                        path:
+                          type: string
+                          description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located."
+                reconcile:
+                  type: object
+                  description: "allow tuning reconciling process"
+                  properties:
+                    runtime:
+                      type: object
+                      description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle"
+                      properties:
+                        reconcileCHIsThreadsNumber:
+                          type: integer
+                          minimum: 1
+                          maximum: 65535
+                          description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default"
+                        reconcileShardsThreadsNumber:
+                          type: integer
+                          minimum: 1
+                          maximum: 65535
+                          description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default"
+                        reconcileShardsMaxConcurrencyPercent:
+                          type: integer
+                          minimum: 0
+                          maximum: 100
+                          description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default."
+                    statefulSet:
+                      type: object
+                      description: "Allow change default behavior for reconciling StatefulSet which generated by clickhouse-operator"
+                      properties:
+                        create:
+                          type: object
+                          description: "Behavior during create StatefulSet"
+                          properties:
+                            onFailure:
+                              type: string
+                              description: |
+                                What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds
+                                Possible options:
+                                1. abort - do nothing, just break the process and wait for admin.
+                                2. delete - delete newly created problematic StatefulSet.
+                                3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet.
+                        update:
+                          type: object
+                          description: "Behavior during update StatefulSet"
+                          properties:
+                            timeout:
+                              type: integer
+                              description: "How many seconds to wait for created/updated StatefulSet to be Ready"
+                            pollInterval:
+                              type: integer
+                              description: "How many seconds to wait between checks for created/updated StatefulSet status"
+                            onFailure:
+                              type: string
+                              description: |
+                                What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds
+                                Possible options:
+                                1. abort - do nothing, just break the process and wait for admin.
+                                2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration.
+                                3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet.
+                    host:
+                      type: object
+                      description: |
+                        Whether the operator during reconcile procedure should wait for a ClickHouse host:
+                          - to be excluded from a ClickHouse cluster
+                          - to complete all running queries
+                          - to be included into a ClickHouse cluster
+                        respectfully before moving forward
+                      properties:
+                        wait:
+                          type: object
+                          properties:
+                            exclude: &TypeStringBool
+                              type: string
+                              description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be excluded from a ClickHouse cluster"
+                              enum:
+                                # List StringBoolXXX constants from model
+                                - ""
+                                - "0"
+                                - "1"
+                                - "False"
+                                - "false"
+                                - "True"
+                                - "true"
+                                - "No"
+                                - "no"
+                                - "Yes"
+                                - "yes"
+                                - "Off"
+                                - "off"
+                                - "On"
+                                - "on"
+                                - "Disable"
+                                - "disable"
+                                - "Enable"
+                                - "enable"
+                                - "Disabled"
+                                - "disabled"
+                                - "Enabled"
+                                - "enabled"
+                            queries:
+                              !!merge <<: *TypeStringBool
+                              description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries"
+                            include:
+                              !!merge <<: *TypeStringBool
+                              description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster"
+                annotation:
+                  type: object
+                  description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources"
+                  properties:
+                    include:
+                      type: array
+                      description: |
+                        When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`,
+                        include annotations with names from the following list
+                      items:
+                        type: string
+                    exclude:
+                      type: array
+                      description: |
+                        When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`,
+                        exclude annotations with names from the following list
+                      items:
+                        type: string
+                label:
+                  type: object
+                  description: "defines which metadata.labels will include or exclude during render StatefulSet, Pod, PVC resources"
+                  properties:
+                    include:
+                      type: array
+                      description: |
+                        When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`,
+                        include labels from the following list
+                      items:
+                        type: string
+                    exclude:
+                      type: array
+                      items:
+                        type: string
+                      description: |
+                        When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`,
+                        exclude labels from the following list
+                    appendScope:
+                      !!merge <<: *TypeStringBool
+                      description: |
+                        Whether to append *Scope* labels to StatefulSet and Pod
+                        - "LabelShardScopeIndex"
+                        - "LabelReplicaScopeIndex"
+                        - "LabelCHIScopeIndex"
+                        - "LabelCHIScopeCycleSize"
+                        - "LabelCHIScopeCycleIndex"
+                        - "LabelCHIScopeCycleOffset"
+                        - "LabelClusterScopeIndex"
+                        - "LabelClusterScopeCycleSize"
+                        - "LabelClusterScopeCycleIndex"
+                        - "LabelClusterScopeCycleOffset"
+                statefulSet:
+                  type: object
+                  description: "define StatefulSet-specific parameters"
+                  properties:
+                    revisionHistoryLimit:
+                      type: integer
+                      description: "revisionHistoryLimit is the maximum number of revisions that will be\nmaintained in the StatefulSet's revision history.                         \nLook details in `statefulset.spec.revisionHistoryLimit`\n"
+                pod:
+                  type: object
+                  description: "define pod specific parameters"
+                  properties:
+                    terminationGracePeriod:
+                      type: integer
+                      description: "Optional duration in seconds the pod needs to terminate gracefully. \nLook details in `pod.spec.terminationGracePeriodSeconds`\n"
+                logger:
+                  type: object
+                  description: "allow setup clickhouse-operator logger behavior"
+                  properties:
+                    logtostderr:
+                      type: string
+                      description: "boolean, allows logs to stderr"
+                    alsologtostderr:
+                      type: string
+                      description: "boolean allows logs to stderr and files both"
+                    v:
+                      type: string
+                      description: "verbosity level of clickhouse-operator log, default - 1 max - 9"
+                    stderrthreshold:
+                      type: string
+                    vmodule:
+                      type: string
+                      description: |
+                        Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level.
+                        Ex.: file*=2 sets the 'V' to 2 in all files with names like file*.
+                    log_backtrace_at:
+                      type: string
+                      description: |
+                        It can be set to a file and line number with a logging line.
+                        Ex.: file.go:123
+                        Each time when this line is being executed, a stack trace will be written to the Info log.

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/_helpers.tpl

@@ -0,0 +1,102 @@
+{{/* vim: set filetype=go-template: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "altinity-clickhouse-operator.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "altinity-clickhouse-operator.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "altinity-clickhouse-operator.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "altinity-clickhouse-operator.labels" -}}
+helm.sh/chart: {{ include "altinity-clickhouse-operator.chart" . }}
+{{ include "altinity-clickhouse-operator.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+{{- if .Values.podLabels }}
+{{ toYaml .Values.podLabels }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "altinity-clickhouse-operator.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "altinity-clickhouse-operator.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "altinity-clickhouse-operator.serviceAccountName" -}}
+    {{ default (include "altinity-clickhouse-operator.fullname" .) .Values.serviceAccount.name }}
+{{- end -}}
+
+{{/*
+Create the tag for the docker image to use
+*/}}
+{{- define "altinity-clickhouse-operator.operator.tag" -}}
+{{- .Values.operator.image.tag | default .Chart.AppVersion -}}
+{{- end -}}
+
+{{/*
+Create the tag for the docker image to use
+*/}}
+{{- define "altinity-clickhouse-operator.metrics.tag" -}}
+{{- .Values.metrics.image.tag | default .Chart.AppVersion -}}
+{{- end -}}
+
+{{/*
+altinity-clickhouse-operator.rawResource will create a resource template that can be
+merged with each item in `.Values.additionalResources`.
+*/}}
+{{- define "altinity-clickhouse-operator.rawResource" -}}
+metadata:
+  labels:
+    {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+{{- end }}
+
+{{/*
+*/}}
+{{- define "altinity-clickhouse-operator.configmap-data" }}
+{{- $root := index . 0 }}
+{{- $data := index . 1 }}
+{{- if not $data -}}
+null
+{{ end }}
+{{- range $k, $v := $data }}
+{{- if not (kindIs "string" $v) }}
+{{- $v = toYaml $v }}
+{{- end }}
+{{- tpl (toYaml (dict $k $v)) $root }}
+{{ end }}
+{{- end }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/additional-resources.yaml

@@ -0,0 +1,5 @@
+{{- $template := fromYaml (include "altinity-clickhouse-operator.rawResource" .) -}}
+{{- range $i, $t := .Values.additionalResources }}
+---
+{{ toYaml (merge (tpl $t $ | fromYaml) $template) -}}
+{{- end }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-secret.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.dashboards.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "altinity-clickhouse-operator.fullname" . }}-dashboards
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+{{- if .Values.dashboards.additionalLabels }}
+    {{- toYaml .Values.dashboards.additionalLabels | nindent 4 }}
+{{- end }}
+{{- with .Values.dashboards.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+{{- end }}
+type: Opaque
+data:
+{{- range $path, $_ := .Files.Glob "files/*.json" }}
+  {{ $path | trimPrefix "files/" }}: {{ $.Files.Get $path | b64enc -}}
+{{ end }}
+{{- end }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml

@@ -0,0 +1,211 @@
+{{- if .Values.rbac.create -}}
+# Specifies either
+#   ClusterRole
+# or
+#   Role
+# to be bound to ServiceAccount.
+# ClusterRole is namespace-less and must have unique name
+# Role is namespace-bound
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "altinity-clickhouse-operator.fullname" . }}
+  #namespace: kube-system
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+  namespace: {{ .Release.Namespace }}
+rules:
+  #
+  # Core API group
+  #
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - services
+      - persistentvolumeclaims
+      - secrets
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
+      - create
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+  #
+  # apps.* resources
+  #
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
+      - create
+      - delete
+  - apiGroups:
+      - apps
+    resources:
+      - replicasets
+    verbs:
+      - get
+      - patch
+      - update
+      - delete
+  # The operator deployment personally, identified by name
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+    resourceNames:
+      - {{ include "altinity-clickhouse-operator.fullname" . }}
+    verbs:
+      - get
+      - patch
+      - update
+      - delete
+  #
+  # policy.* resources
+  #
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
+      - create
+      - delete
+  #
+  # apiextensions
+  #
+  - apiGroups:
+      - apiextensions.k8s.io
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - get
+      - list
+  # clickhouse - related resources
+  - apiGroups:
+      - clickhouse.altinity.com
+    #
+    # The operators specific Custom Resources
+    #
+
+    resources:
+      - clickhouseinstallations
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+      - update
+      - delete
+  - apiGroups:
+      - clickhouse.altinity.com
+    resources:
+      - clickhouseinstallationtemplates
+      - clickhouseoperatorconfigurations
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - clickhouse.altinity.com
+    resources:
+      - clickhouseinstallations/finalizers
+      - clickhouseinstallationtemplates/finalizers
+      - clickhouseoperatorconfigurations/finalizers
+    verbs:
+      - update
+  - apiGroups:
+      - clickhouse.altinity.com
+    resources:
+      - clickhouseinstallations/status
+      - clickhouseinstallationtemplates/status
+      - clickhouseoperatorconfigurations/status
+    verbs:
+      - get
+      - update
+      - patch
+      - create
+      - delete
+  # clickhouse-keeper - related resources
+  - apiGroups:
+      - clickhouse-keeper.altinity.com
+    resources:
+      - clickhousekeeperinstallations
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+      - update
+      - delete
+  - apiGroups:
+      - clickhouse-keeper.altinity.com
+    resources:
+      - clickhousekeeperinstallations/finalizers
+    verbs:
+      - update
+  - apiGroups:
+      - clickhouse-keeper.altinity.com
+    resources:
+      - clickhousekeeperinstallations/status
+    verbs:
+      - get
+      - update
+      - patch
+      - create
+      - delete
+{{- end }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml

@@ -0,0 +1,23 @@
+{{- if .Values.rbac.create -}}
+# Specifies either
+#   ClusterRoleBinding between ClusterRole and ServiceAccount.
+# or
+#   RoleBinding between Role and ServiceAccount.
+# ClusterRoleBinding is namespace-less and must have unique name
+# RoleBinding is namespace-bound
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "altinity-clickhouse-operator.fullname" . }}
+  #namespace: kube-system
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "altinity-clickhouse-operator.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "altinity-clickhouse-operator.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml

@@ -0,0 +1,13 @@
+# Template Parameters:
+#
+# NAME=etc-clickhouse-operator-confd-files
+# NAMESPACE=kube-system
+# COMMENT=
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-confd-files" (include "altinity-clickhouse-operator.fullname" .) }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.confdFiles) | nindent 2 }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml

@@ -0,0 +1,13 @@
+# Template Parameters:
+#
+# NAME=etc-clickhouse-operator-configd-files
+# NAMESPACE=kube-system
+# COMMENT=
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-configd-files" (include "altinity-clickhouse-operator.fullname" .) }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.configdFiles) | nindent 2 }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml

@@ -0,0 +1,13 @@
+# Template Parameters:
+#
+# NAME=etc-clickhouse-operator-files
+# NAMESPACE=kube-system
+# COMMENT=
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-files" (include "altinity-clickhouse-operator.fullname" .) }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.files) | nindent 2 }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml

@@ -0,0 +1,13 @@
+# Template Parameters:
+#
+# NAME=etc-clickhouse-operator-templatesd-files
+# NAMESPACE=kube-system
+# COMMENT=
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-templatesd-files" (include "altinity-clickhouse-operator.fullname" .) }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.templatesdFiles) | nindent 2 }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml

@@ -0,0 +1,13 @@
+# Template Parameters:
+#
+# NAME=etc-clickhouse-operator-usersd-files
+# NAMESPACE=kube-system
+# COMMENT=
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-usersd-files" (include "altinity-clickhouse-operator.fullname" .) }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.usersdFiles) | nindent 2 }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml

@@ -0,0 +1,195 @@
+# Template Parameters:
+#
+# NAMESPACE=kube-system
+# COMMENT=
+# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.4
+# OPERATOR_IMAGE_PULL_POLICY=Always
+# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.4
+# METRICS_EXPORTER_IMAGE_PULL_POLICY=Always
+#
+# Setup Deployment for clickhouse-operator
+# Deployment would be created in kubectl-specified namespace
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: {{ include "altinity-clickhouse-operator.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {{ include "altinity-clickhouse-operator.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 8 }}
+      annotations:
+        {{ toYaml .Values.podAnnotations | nindent 8 }}
+        checksum/files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-files.yaml") . | sha256sum }}
+        checksum/confd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml") . | sha256sum }}
+        checksum/configd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml") . | sha256sum }}
+        checksum/templatesd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml") . | sha256sum }}
+        checksum/usersd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml") . | sha256sum }}
+    spec:
+      serviceAccountName: {{ include "altinity-clickhouse-operator.serviceAccountName" . }}
+      volumes:
+        - name: etc-clickhouse-operator-folder
+          configMap:
+            name: {{ include "altinity-clickhouse-operator.fullname" . }}-files
+        - name: etc-clickhouse-operator-confd-folder
+          configMap:
+            name: {{ include "altinity-clickhouse-operator.fullname" . }}-confd-files
+        - name: etc-clickhouse-operator-configd-folder
+          configMap:
+            name: {{ include "altinity-clickhouse-operator.fullname" . }}-configd-files
+        - name: etc-clickhouse-operator-templatesd-folder
+          configMap:
+            name: {{ include "altinity-clickhouse-operator.fullname" . }}-templatesd-files
+        - name: etc-clickhouse-operator-usersd-folder
+          configMap:
+            name: {{ include "altinity-clickhouse-operator.fullname" . }}-usersd-files
+      containers:
+        - name: {{ .Chart.Name }}
+          image: {{ .Values.operator.image.repository }}:{{ include "altinity-clickhouse-operator.operator.tag" . }}
+          imagePullPolicy: {{ .Values.operator.image.pullPolicy }}
+          volumeMounts:
+            - name: etc-clickhouse-operator-folder
+              mountPath: /etc/clickhouse-operator
+            - name: etc-clickhouse-operator-confd-folder
+              mountPath: /etc/clickhouse-operator/conf.d
+            - name: etc-clickhouse-operator-configd-folder
+              mountPath: /etc/clickhouse-operator/config.d
+            - name: etc-clickhouse-operator-templatesd-folder
+              mountPath: /etc/clickhouse-operator/templates.d
+            - name: etc-clickhouse-operator-usersd-folder
+              mountPath: /etc/clickhouse-operator/users.d
+          env:
+            # Pod-specific
+            # spec.nodeName: ip-172-20-52-62.ec2.internal
+            - name: OPERATOR_POD_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # metadata.name: clickhouse-operator-6f87589dbb-ftcsf
+            - name: OPERATOR_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            # metadata.namespace: kube-system
+            - name: OPERATOR_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            # status.podIP: 100.96.3.2
+            - name: OPERATOR_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            # spec.serviceAccount: clickhouse-operator
+            # spec.serviceAccountName: clickhouse-operator
+            - name: OPERATOR_POD_SERVICE_ACCOUNT
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.serviceAccountName
+            # Container-specific
+            - name: OPERATOR_CONTAINER_CPU_REQUEST
+              valueFrom:
+                resourceFieldRef:
+                  containerName: {{ .Chart.Name }}
+                  resource: requests.cpu
+            - name: OPERATOR_CONTAINER_CPU_LIMIT
+              valueFrom:
+                resourceFieldRef:
+                  containerName: {{ .Chart.Name }}
+                  resource: limits.cpu
+            - name: OPERATOR_CONTAINER_MEM_REQUEST
+              valueFrom:
+                resourceFieldRef:
+                  containerName: {{ .Chart.Name }}
+                  resource: requests.memory
+            - name: OPERATOR_CONTAINER_MEM_LIMIT
+              valueFrom:
+                resourceFieldRef:
+                  containerName: {{ .Chart.Name }}
+                  resource: limits.memory
+            {{ with .Values.operator.env }}{{ toYaml . | nindent 12 }}{{ end }}
+          ports:
+            - containerPort: 9999
+              name: metrics
+          resources: {{ toYaml .Values.operator.resources | nindent 12 }}
+          securityContext: {{ toYaml .Values.operator.containerSecurityContext | nindent 12 }}
+{{ if .Values.metrics.enabled }}
+        - name: metrics-exporter
+          image: {{ .Values.metrics.image.repository }}:{{ include "altinity-clickhouse-operator.metrics.tag" . }}
+          imagePullPolicy: {{ .Values.metrics.image.pullPolicy }}
+          volumeMounts:
+            - name: etc-clickhouse-operator-folder
+              mountPath: /etc/clickhouse-operator
+            - name: etc-clickhouse-operator-confd-folder
+              mountPath: /etc/clickhouse-operator/conf.d
+            - name: etc-clickhouse-operator-configd-folder
+              mountPath: /etc/clickhouse-operator/config.d
+            - name: etc-clickhouse-operator-templatesd-folder
+              mountPath: /etc/clickhouse-operator/templates.d
+            - name: etc-clickhouse-operator-usersd-folder
+              mountPath: /etc/clickhouse-operator/users.d
+          env:
+            # Pod-specific
+            # spec.nodeName: ip-172-20-52-62.ec2.internal
+            - name: OPERATOR_POD_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # metadata.name: clickhouse-operator-6f87589dbb-ftcsf
+            - name: OPERATOR_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            # metadata.namespace: kube-system
+            - name: OPERATOR_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            # status.podIP: 100.96.3.2
+            - name: OPERATOR_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            # spec.serviceAccount: clickhouse-operator
+            # spec.serviceAccountName: clickhouse-operator
+            - name: OPERATOR_POD_SERVICE_ACCOUNT
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.serviceAccountName
+            # Container-specific
+            - name: OPERATOR_CONTAINER_CPU_REQUEST
+              valueFrom:
+                resourceFieldRef:
+                  containerName: {{ .Chart.Name }}
+                  resource: requests.cpu
+            - name: OPERATOR_CONTAINER_CPU_LIMIT
+              valueFrom:
+                resourceFieldRef:
+                  containerName: {{ .Chart.Name }}
+                  resource: limits.cpu
+            - name: OPERATOR_CONTAINER_MEM_REQUEST
+              valueFrom:
+                resourceFieldRef:
+                  containerName: {{ .Chart.Name }}
+                  resource: requests.memory
+            - name: OPERATOR_CONTAINER_MEM_LIMIT
+              valueFrom:
+                resourceFieldRef:
+                  containerName: {{ .Chart.Name }}
+                  resource: limits.memory
+            {{ with .Values.metrics.env }}{{ toYaml . | nindent 12 }}{{ end }}
+          ports:
+            - containerPort: 8888
+              name: metrics
+          resources: {{ toYaml .Values.metrics.resources | nindent 12 }}
+          securityContext: {{ toYaml .Values.metrics.containerSecurityContext | nindent 12 }}
+{{ end }}
+      imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }}
+      nodeSelector: {{ toYaml .Values.nodeSelector | nindent 8 }}
+      affinity: {{ toYaml .Values.affinity | nindent 8 }}
+      tolerations: {{ toYaml .Values.tolerations | nindent 8 }}
+      securityContext: {{ toYaml .Values.podSecurityContext | nindent 8 }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.secret.create -}}
+#
+# Template parameters available:
+#   NAMESPACE=kube-system
+#   COMMENT=
+#   OPERATOR_VERSION=0.23.4
+#   CH_USERNAME_SECRET_PLAIN=clickhouse_operator
+#   CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password
+#
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "altinity-clickhouse-operator.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+type: Opaque
+data:
+  username: {{ .Values.secret.username | b64enc }}
+  password: {{ .Values.secret.password | b64enc }}
+{{- end -}}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml

@@ -0,0 +1,23 @@
+# Template Parameters:
+#
+# NAMESPACE=kube-system
+# COMMENT=
+#
+# Setup ClusterIP Service to provide monitoring metrics for Prometheus
+# Service would be created in kubectl-specified namespace
+# In order to get access outside of k8s it should be exposed as:
+# kubectl --namespace prometheus port-forward service/prometheus 9090
+# and point browser to localhost:9090
+kind: Service
+apiVersion: v1
+metadata:
+  name: {{ printf "%s-metrics" (include "altinity-clickhouse-operator.fullname" .) }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+spec:
+  ports:
+    - port: 8888
+      name: clickhouse-metrics
+    - port: 9999
+      name: operator-metrics
+  selector: {{ include "altinity-clickhouse-operator.selectorLabels" . | nindent 4 }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.serviceAccount.create -}}
+# Template Parameters:
+#
+# COMMENT=
+# NAMESPACE=kube-system
+# NAME=clickhouse-operator
+#
+# Setup ServiceAccount
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "altinity-clickhouse-operator.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+  annotations: {{ toYaml .Values.serviceAccount.annotations | nindent 4 }}
+
+# Template Parameters:
+#
+# NAMESPACE=kube-system
+# COMMENT=#
+# ROLE_KIND=ClusterRole
+# ROLE_NAME=clickhouse-operator-kube-system
+# ROLE_BINDING_KIND=ClusterRoleBinding
+# ROLE_BINDING_NAME=clickhouse-operator-kube-system
+#
+{{- end -}}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/servicemonitor.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ printf "%s-clickhouse-metrics" (include "altinity-clickhouse-operator.fullname" .) }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }}
+  {{- if .Values.serviceMonitor.additionalLabels }}
+    {{- toYaml .Values.serviceMonitor.additionalLabels | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+    - port: clickhouse-metrics # 8888
+    - port: operator-metrics # 9999
+  selector:
+    matchLabels:
+      {{- include "altinity-clickhouse-operator.selectorLabels" . | nindent 6 }}
+{{- end }}

packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/values.yaml

@@ -0,0 +1,670 @@
+operator:
+  image:
+    # operator.image.repository -- image repository
+    repository: altinity/clickhouse-operator
+    # operator.image.tag -- image tag (chart's appVersion value will be used if not set)
+    tag: ""
+    # operator.image.pullPolicy -- image pull policy
+    pullPolicy: IfNotPresent
+  containerSecurityContext: {}
+  # operator.resources -- custom resource configuration, look `kubectl explain pod.spec.containers.resources` for details
+  resources: {}
+  #  limits:
+  #    cpu: 100m
+  #    memory: 128Mi
+  #  requests:
+  #    cpu: 100m
+  #    memory: 128Mi
+
+  # operator.env -- additional environment variables for the clickhouse-operator container in deployment
+  # possible format value [{"name": "SAMPLE", "value": "text"}]
+  env: []
+metrics:
+  enabled: true
+  image:
+    # metrics.image.repository -- image repository
+    repository: altinity/metrics-exporter
+    # metrics.image.tag -- image tag (chart's appVersion value will be used if not set)
+    tag: ""
+    # metrics.image.pullPolicy -- image pull policy
+    pullPolicy: IfNotPresent
+  containerSecurityContext: {}
+  # metrics.resources -- custom resource configuration
+  resources: {}
+  #  limits:
+  #    cpu: 100m
+  #    memory: 128Mi
+  #  requests:
+  #    cpu: 100m
+  #    memory: 128Mi
+
+  # metrics.env -- additional environment variables for the deployment of metrics-exporter containers
+  # possible format value [{"name": "SAMPLE", "value": "text"}]
+  env: []
+# imagePullSecrets -- image pull secret for private images in clickhouse-operator pod
+#  possible value format [{"name":"your-secret-name"}]
+#  look `kubectl explain pod.spec.imagePullSecrets` for details
+imagePullSecrets: []
+# podLabels -- labels to add to the clickhouse-operator pod
+podLabels: {}
+# podAnnotations -- annotations to add to the clickhouse-operator pod, look `kubectl explain pod.spec.annotations` for details
+podAnnotations:
+  prometheus.io/port: '8888'
+  prometheus.io/scrape: 'true'
+  clickhouse-operator-metrics/port: '9999'
+  clickhouse-operator-metrics/scrape: 'true'
+# nameOverride -- override name of the chart
+nameOverride: ""
+# fullnameOverride -- full name of the chart.
+fullnameOverride: ""
+serviceAccount:
+  # serviceAccount.create -- specifies whether a service account should be created
+  create: true
+  # serviceAccount.annotations -- annotations to add to the service account
+  annotations: {}
+  # serviceAccount.name -- the name of the service account to use; if not set and create is true, a name is generated using the fullname template
+  name:
+rbac:
+  # rbac.create -- specifies whether cluster roles and cluster role bindings should be created
+  create: true
+secret:
+  # secret.create -- create a secret with operator credentials
+  create: true
+  # secret.username -- operator credentials username
+  username: clickhouse_operator
+  # secret.password -- operator credentials password
+  password: clickhouse_operator_password
+# nodeSelector -- node for scheduler pod assignment, look `kubectl explain pod.spec.nodeSelector` for details
+nodeSelector: {}
+# tolerations -- tolerations for scheduler pod assignment, look `kubectl explain pod.spec.tolerations` for details
+tolerations: []
+# affinity -- affinity for scheduler pod assignment, look `kubectl explain pod.spec.affinity` for details
+affinity: {}
+# podSecurityContext - operator deployment SecurityContext, look `kubectl explain pod.spec.securityContext` for details
+podSecurityContext: {}
+serviceMonitor:
+  # serviceMonitor.enabled -- ServiceMonitor Custom resource is created for a (prometheus-operator)[https://github.com/prometheus-operator/prometheus-operator]
+  enabled: false
+  # serviceMonitor.additionalLabels -- additional labels for service monitor
+  additionalLabels: {}
+# configs -- clickhouse-operator configs
+# @default -- check the values.yaml file for the config content, auto-generated from latest operator release
+configs:
+  confdFiles: null
+  configdFiles:
+    01-clickhouse-01-listen.xml: |
+      <!-- IMPORTANT -->
+      <!-- This file is auto-generated -->
+      <!-- Do not edit this file - all changes would be lost -->
+      <!-- Edit appropriate template in the following folder: -->
+      <!-- deploy/builder/templates-config -->
+      <!-- IMPORTANT -->
+      <yandex>
+          <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
+          <listen_host>::</listen_host>
+          <listen_host>0.0.0.0</listen_host>
+          <listen_try>1</listen_try>
+      </yandex>
+    01-clickhouse-02-logger.xml: |
+      <!-- IMPORTANT -->
+      <!-- This file is auto-generated -->
+      <!-- Do not edit this file - all changes would be lost -->
+      <!-- Edit appropriate template in the following folder: -->
+      <!-- deploy/builder/templates-config -->
+      <!-- IMPORTANT -->
+      <yandex>
+          <logger>
+              <!-- Possible levels: https://github.com/pocoproject/poco/blob/devel/Foundation/include/Poco/Logger.h#L439 -->
+              <level>debug</level>
+              <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+              <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+              <size>1000M</size>
+              <count>10</count>
+              <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
+              <console>1</console>
+          </logger>
+      </yandex>
+    01-clickhouse-03-query_log.xml: |
+      <!-- IMPORTANT -->
+      <!-- This file is auto-generated -->
+      <!-- Do not edit this file - all changes would be lost -->
+      <!-- Edit appropriate template in the following folder: -->
+      <!-- deploy/builder/templates-config -->
+      <!-- IMPORTANT -->
+      <yandex>
+          <query_log replace="1">
+              <database>system</database>
+              <table>query_log</table>
+              <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
+              <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+          </query_log>
+          <query_thread_log remove="1"/>
+      </yandex>
+    01-clickhouse-04-part_log.xml: |
+      <!-- IMPORTANT -->
+      <!-- This file is auto-generated -->
+      <!-- Do not edit this file - all changes would be lost -->
+      <!-- Edit appropriate template in the following folder: -->
+      <!-- deploy/builder/templates-config -->
+      <!-- IMPORTANT -->
+      <yandex>
+          <part_log replace="1">
+              <database>system</database>
+              <table>part_log</table>
+              <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
+              <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+          </part_log>
+      </yandex>
+    01-clickhouse-05-trace_log.xml: |-
+      <!-- IMPORTANT -->
+      <!-- This file is auto-generated -->
+      <!-- Do not edit this file - all changes would be lost -->
+      <!-- Edit appropriate template in the following folder: -->
+      <!-- deploy/builder/templates-config -->
+      <!-- IMPORTANT -->
+      <yandex>
+          <trace_log replace="1">
+              <database>system</database>
+              <table>trace_log</table>
+              <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
+              <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+          </trace_log>
+      </yandex>
+  files:
+    config.yaml:
+      # IMPORTANT
+      # This file is auto-generated
+      # Do not edit this file - all changes would be lost
+      # Edit appropriate template in the following folder:
+      # deploy/builder/templates-config
+      # IMPORTANT
+      #
+      # Template parameters available:
+      #   WATCH_NAMESPACES=
+      #   CH_USERNAME_PLAIN=
+      #   CH_PASSWORD_PLAIN=
+      #   CH_CREDENTIALS_SECRET_NAMESPACE=
+      #   CH_CREDENTIALS_SECRET_NAME=clickhouse-operator
+      #   VERBOSITY=1
+
+      ################################################
+      ##
+      ## Watch section
+      ##
+      ################################################
+      watch:
+        # List of namespaces where clickhouse-operator watches for events.
+        # Concurrently running operators should watch on different namespaces.
+        # IMPORTANT
+        # Regexp is applicable.
+        #namespaces: ["dev", "test"]
+        namespaces: []
+      clickhouse:
+        configuration:
+          ################################################
+          ##
+          ## Configuration files section
+          ##
+          ################################################
+          file:
+            # Each 'path' can be either absolute or relative.
+            # In case path is absolute - it is used as is
+            # In case path is relative - it is relative to the folder where configuration file you are reading right now is located.
+            path:
+              # Path to the folder where ClickHouse configuration files common for all instances within a CHI are located.
+              common: config.d
+              # Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located.
+              host: conf.d
+              # Path to the folder where ClickHouse configuration files with users' settings are located.
+              # Files are common for all instances within a CHI.
+              user: users.d
+          ################################################
+          ##
+          ## Configuration users section
+          ##
+          ################################################
+          user:
+            # Default settings for user accounts, created by the operator.
+            # IMPORTANT. These are not access credentials or settings for 'default' user account,
+            # it is a template for filling out missing fields for all user accounts to be created by the operator,
+            # with the following EXCEPTIONS:
+            # 1. 'default' user account DOES NOT use provided password, but uses all the rest of the fields.
+            #    Password for 'default' user account has to be provided explicitly, if to be used.
+            # 2. CHOP user account DOES NOT use:
+            #    - profile setting. It uses predefined profile called 'clickhouse_operator'
+            #    - quota setting. It uses empty quota name.
+            #    - networks IP setting. Operator specifies 'networks/ip' user setting to match operators' pod IP only.
+            #    - password setting. Password for CHOP account is used from 'clickhouse.access.*' section
+            default:
+              # Default values for ClickHouse user account(s) created by the operator
+              #   1. user/profile - string
+              #   2. user/quota - string
+              #   3. user/networks/ip - multiple strings
+              #   4. user/password - string
+              # These values can be overwritten on per-user basis.
+              profile: "default"
+              quota: "default"
+              networksIP:
+                - "::1"
+                - "127.0.0.1"
+              password: "default"
+          ################################################
+          ##
+          ## Configuration network section
+          ##
+          ################################################
+          network:
+            # Default host_regexp to limit network connectivity from outside
+            hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
+        ################################################
+        ##
+        ## Configuration restart policy section
+        ## Configuration restart policy describes what configuration changes require ClickHouse restart
+        ##
+        ################################################
+        configurationRestartPolicy:
+          rules:
+            # IMPORTANT!
+            # Special version of "*" - default version - has to satisfy all ClickHouse versions.
+            # Default version will also be used in case ClickHouse version is unknown.
+            # ClickHouse version may be unknown due to host being down - for example, because of incorrect "settings" section.
+            # ClickHouse is not willing to start in case incorrect/unknown settings are provided in config file.
+            - version: "*"
+              rules:
+                # see https://kb.altinity.com/altinity-kb-setup-and-maintenance/altinity-kb-server-config-files/#server-config-configxml-sections-which-dont-require-restart
+                # to be replaced with "select * from system.server_settings where changeable_without_restart = 'No'"
+                - settings/*: "yes"
+                # single values
+                - settings/access_control_path: "no"
+                - settings/dictionaries_config: "no"
+                - settings/max_server_memory_*: "no"
+                - settings/max_*_to_drop: "no"
+                - settings/max_concurrent_queries: "no"
+                - settings/models_config: "no"
+                - settings/user_defined_executable_functions_config: "no"
+                # structured XML
+                - settings/logger/*: "no"
+                - settings/macros/*: "no"
+                - settings/remote_servers/*: "no"
+                - settings/user_directories/*: "no"
+                - zookeeper/*: "yes"
+                - files/*.xml: "yes"
+                - files/config.d/*.xml: "yes"
+                - files/config.d/*dict*.xml: "no"
+                # exceptions in default profile
+                - profiles/default/background_*_pool_size: "yes"
+                - profiles/default/max_*_for_server: "yes"
+            - version: "21.*"
+              rules:
+                - settings/logger: "yes"
+        #################################################
+        ##
+        ## Access to ClickHouse instances
+        ##
+        ################################################
+        access:
+          # Possible values for 'scheme' are:
+          #   1. http - force http to be used to connect to ClickHouse instances
+          #   2. https - force https to be used to connect to ClickHouse instances
+          #   3. auto - either http or https is selected based on open ports
+          scheme: "auto"
+          # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances.
+          # These credentials are used for:
+          #   1. Metrics requests
+          #   2. Schema maintenance
+          #   3. DROP DNS CACHE
+          # User with these credentials can be specified in additional ClickHouse .xml config files,
+          # located in 'clickhouse.configuration.file.path.user' folder
+          username: ""
+          password: ""
+          rootCA: ""
+          # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances.
+          # Can be used instead of explicitly specified username and password available in sections:
+          #   - clickhouse.access.username
+          #   - clickhouse.access.password
+          # Secret should have two keys:
+          #   1. username
+          #   2. password
+          secret:
+            # Empty `namespace` means that k8s secret would be looked in the same namespace where operator's pod is running.
+            namespace: ""
+            # Empty `name` means no k8s Secret would be looked for
+            name: '{{ include "altinity-clickhouse-operator.fullname" . }}'
+          # Port where to connect to ClickHouse instances to
+          port: 8123
+          # Timeouts used to limit connection and queries from the operator to ClickHouse instances
+          # Specified in seconds.
+          timeouts:
+            # Timout to setup connection from the operator to ClickHouse instances. In seconds.
+            connect: 1
+            # Timout to perform SQL query from the operator to ClickHouse instances. In seconds.
+            query: 4
+        #################################################
+        ##
+        ## Metrics collection
+        ##
+        ################################################
+        metrics:
+          # Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances
+          # Specified in seconds.
+          timeouts:
+            # Timeout used to limit metrics collection request. In seconds.
+            # Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle.
+            # All collected metrics are returned.
+            collect: 9
+      ################################################
+      ##
+      ## Template(s) management section
+      ##
+      ################################################
+      template:
+        chi:
+          # CHI template updates handling policy
+          # Possible policy values:
+          #   - ReadOnStart. Accept CHIT updates on the operators start only.
+          #   - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI
+          policy: ApplyOnNextReconcile
+          # Path to the folder where ClickHouseInstallation templates .yaml manifests are located.
+          # Templates are added to the list of all templates and used when CHI is reconciled.
+          # Templates are applied in sorted alpha-numeric order.
+          path: templates.d
+      ################################################
+      ##
+      ## Reconcile section
+      ##
+      ################################################
+      reconcile:
+        # Reconcile runtime settings
+        runtime:
+          # Max number of concurrent CHI reconciles in progress
+          reconcileCHIsThreadsNumber: 10
+          # The operator reconciles shards concurrently in each CHI with the following limitations:
+          #   1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently
+          #      can not be greater than 'reconcileShardsThreadsNumber'.
+          #   2. Percentage of shards being reconciled (and thus having hosts down) in each CHI concurrently
+          #      can not be greater than 'reconcileShardsMaxConcurrencyPercent'.
+          #   3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward.
+          # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage
+
+          # Max number of concurrent shard reconciles within one CHI in progress
+          reconcileShardsThreadsNumber: 5
+          # Max percentage of concurrent shard reconciles within one CHI in progress
+          reconcileShardsMaxConcurrencyPercent: 50
+        # Reconcile StatefulSet scenario
+        statefulSet:
+          # Create StatefulSet scenario
+          create:
+            # What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds
+            # Possible options:
+            # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
+            #    do not try to fix or delete or update it, just abort reconcile cycle.
+            #    Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
+            # 2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards.
+            # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
+            onFailure: ignore
+          # Update StatefulSet scenario
+          update:
+            # How many seconds to wait for created/updated StatefulSet to be 'Ready'
+            timeout: 300
+            # How many seconds to wait between checks/polls for created/updated StatefulSet status
+            pollInterval: 5
+            # What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds
+            # Possible options:
+            # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
+            #    do not try to fix or delete or update it, just abort reconcile cycle.
+            #    Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
+            # 2. rollback - delete Pod and rollback StatefulSet to previous Generation.
+            #    Pod would be recreated by StatefulSet based on rollback-ed StatefulSet configuration.
+            #    Follow 'abort' path afterwards.
+            # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
+            onFailure: abort
+        # Reconcile Host scenario
+        host:
+          # Whether the operator during reconcile procedure should wait for a ClickHouse host:
+          #   - to be excluded from a ClickHouse cluster
+          #   - to complete all running queries
+          #   - to be included into a ClickHouse cluster
+          # respectfully before moving forward
+          wait:
+            exclude: true
+            queries: true
+            include: false
+      ################################################
+      ##
+      ## Annotations management section
+      ##
+      ################################################
+      annotation:
+        # Applied when:
+        #  1. Propagating annotations from the CHI's `metadata.annotations` to child objects' `metadata.annotations`,
+        #  2. Propagating annotations from the CHI Template's `metadata.annotations` to CHI's `metadata.annotations`,
+        # Include annotations from the following list:
+        # Applied only when not empty. Empty list means "include all, no selection"
+        include: []
+        # Exclude annotations from the following list:
+        exclude: []
+      ################################################
+      ##
+      ## Labels management section
+      ##
+      ################################################
+      label:
+        # Applied when:
+        #  1. Propagating labels from the CHI's `metadata.labels` to child objects' `metadata.labels`,
+        #  2. Propagating labels from the CHI Template's `metadata.labels` to CHI's `metadata.labels`,
+        # Include labels from the following list:
+        # Applied only when not empty. Empty list means "include all, no selection"
+        include: []
+        # Exclude labels from the following list:
+        # Applied only when not empty. Empty list means "nothing to exclude, no selection"
+        exclude: []
+        # Whether to append *Scope* labels to StatefulSet and Pod.
+        # Full list of available *scope* labels check in 'labeler.go'
+        #  LabelShardScopeIndex
+        #  LabelReplicaScopeIndex
+        #  LabelCHIScopeIndex
+        #  LabelCHIScopeCycleSize
+        #  LabelCHIScopeCycleIndex
+        #  LabelCHIScopeCycleOffset
+        #  LabelClusterScopeIndex
+        #  LabelClusterScopeCycleSize
+        #  LabelClusterScopeCycleIndex
+        #  LabelClusterScopeCycleOffset
+        appendScope: "no"
+      ################################################
+      ##
+      ## StatefulSet management section
+      ##
+      ################################################
+      statefulSet:
+        revisionHistoryLimit: 0
+      ################################################
+      ##
+      ## Pod management section
+      ##
+      ################################################
+      pod:
+        # Grace period for Pod termination.
+        # How many seconds to wait between sending
+        # SIGTERM and SIGKILL during Pod termination process.
+        # Increase this number is case of slow shutdown.
+        terminationGracePeriod: 30
+      ################################################
+      ##
+      ## Log parameters section
+      ##
+      ################################################
+      logger:
+        logtostderr: "true"
+        alsologtostderr: "false"
+        v: "1"
+        stderrthreshold: ""
+        vmodule: ""
+        log_backtrace_at: ""
+  templatesdFiles:
+    001-templates.json.example: |
+      {
+        "apiVersion": "clickhouse.altinity.com/v1",
+        "kind": "ClickHouseInstallationTemplate",
+        "metadata": {
+          "name": "01-default-volumeclaimtemplate"
+        },
+        "spec": {
+          "templates": {
+            "volumeClaimTemplates": [
+              {
+                "name": "chi-default-volume-claim-template",
+                "spec": {
+                  "accessModes": [
+                    "ReadWriteOnce"
+                  ],
+                  "resources": {
+                    "requests": {
+                      "storage": "2Gi"
+                    }
+                  }
+                }
+              }
+            ],
+            "podTemplates": [
+              {
+                "name": "chi-default-oneperhost-pod-template",
+                "distribution": "OnePerHost",
+                "spec": {
+                  "containers" : [
+                    {
+                      "name": "clickhouse",
+                      "image": "clickhouse/clickhouse-server:23.8",
+                      "ports": [
+                        {
+                          "name": "http",
+                          "containerPort": 8123
+                        },
+                        {
+                          "name": "client",
+                          "containerPort": 9000
+                        },
+                        {
+                          "name": "interserver",
+                          "containerPort": 9009
+                        }
+                      ]
+                    }
+                  ]
+                }
+              }
+            ]
+          }
+        }
+      }
+    default-pod-template.yaml.example: |
+      apiVersion: "clickhouse.altinity.com/v1"
+      kind: "ClickHouseInstallationTemplate"
+      metadata:
+        name: "default-oneperhost-pod-template"
+      spec:
+        templates:
+          podTemplates:
+            - name: default-oneperhost-pod-template
+              distribution: "OnePerHost"
+    default-storage-template.yaml.example: |
+      apiVersion: "clickhouse.altinity.com/v1"
+      kind: "ClickHouseInstallationTemplate"
+      metadata:
+        name: "default-storage-template-2Gi"
+      spec:
+        templates:
+          volumeClaimTemplates:
+            - name: default-storage-template-2Gi
+              spec:
+                accessModes:
+                  - ReadWriteOnce
+                resources:
+                  requests:
+                    storage: 2Gi
+    readme: |-
+      Templates in this folder are packaged with an operator and available via 'useTemplate'
+  usersdFiles:
+    01-clickhouse-operator-profile.xml: |
+      <!-- IMPORTANT -->
+      <!-- This file is auto-generated -->
+      <!-- Do not edit this file - all changes would be lost -->
+      <!-- Edit appropriate template in the following folder: -->
+      <!-- deploy/builder/templates-config -->
+      <!-- IMPORTANT -->
+      <!--
+      #
+      # Template parameters available:
+      #
+      -->
+      <yandex>
+          <!-- clickhouse-operator user is generated by the operator based on config.yaml in runtime -->
+          <profiles>
+              <clickhouse_operator>
+                  <log_queries>0</log_queries>
+                  <skip_unavailable_shards>1</skip_unavailable_shards>
+                  <http_connection_timeout>10</http_connection_timeout>
+                  <max_concurrent_queries_for_all_users>0</max_concurrent_queries_for_all_users>
+                  <os_thread_priority>0</os_thread_priority>
+              </clickhouse_operator>
+          </profiles>
+      </yandex>
+    02-clickhouse-default-profile.xml: |-
+      <!-- IMPORTANT -->
+      <!-- This file is auto-generated -->
+      <!-- Do not edit this file - all changes would be lost -->
+      <!-- Edit appropriate template in the following folder: -->
+      <!-- deploy/builder/templates-config -->
+      <!-- IMPORTANT -->
+      <yandex>
+        <profiles>
+          <default>
+            <os_thread_priority>2</os_thread_priority>
+            <log_queries>1</log_queries>
+            <connect_timeout_with_failover_ms>1000</connect_timeout_with_failover_ms>
+            <distributed_aggregation_memory_efficient>1</distributed_aggregation_memory_efficient>
+            <parallel_view_processing>1</parallel_view_processing>
+            <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>
+            <load_balancing>nearest_hostname</load_balancing>
+            <prefer_localhost_replica>0</prefer_localhost_replica>
+            <!-- materialize_ttl_recalculate_only>1</materialize_ttl_recalculate_only> 21.10 and above -->
+          </default>
+        </profiles>
+      </yandex>
+# additionalResources -- list of additional resources to create (are processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator, look `kubectl explain chi` for details
+additionalResources: []
+#  - |
+#    apiVersion: v1
+#    kind: ConfigMap
+#    metadata:
+#      name: {{ include "altinity-clickhouse-operator.fullname" . }}-cm
+#      namespace: {{ .Release.Namespace }}
+#  - |
+#     apiVersion: v1
+#     kind: Secret
+#     metadata:
+#       name: {{ include "altinity-clickhouse-operator.fullname" . }}-s
+#       namespace: {{ .Release.Namespace }}
+#     stringData:
+#       mykey: my-value
+#  - |
+#     apiVersion: clickhouse.altinity.com/v1
+#     kind: ClickHouseInstallation
+#     metadata:
+#       name: {{ include "altinity-clickhouse-operator.fullname" . }}-chi
+#       namespace: {{ .Release.Namespace }}
+#     spec:
+#       configuration:
+#         clusters:
+#         - name: default
+#           layout:
+#             shardsCount: 1
+dashboards:
+  # dashboards.enabled -- provision grafana dashboards as secrets (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-6.33.1/charts/grafana/values.yaml#L679 )
+  enabled: false
+  # dashboards.additionalLabels -- labels to add to a secret with dashboards
+  additionalLabels:
+    grafana_dashboard: ""
+  # dashboards.annotations -- annotations to add to a secret with dashboards
+  annotations: {}
+  grafana_folder: clickhouse