From 730584bd157a05e90ac05140f8951dcfd0ba2b4b Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 28 Jul 2025 16:18:07 +0300 Subject: [PATCH 01/30] [seaweed] add tests Signed-off-by: IvanHunters --- hack/e2e-apps/bucket.bats | 47 +++++++++++++++++++ hack/e2e-install-cozystack.bats | 6 +-- .../testing/images/e2e-sandbox/Dockerfile | 3 +- 3 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 hack/e2e-apps/bucket.bats diff --git a/hack/e2e-apps/bucket.bats b/hack/e2e-apps/bucket.bats new file mode 100644 index 00000000..2621812a --- /dev/null +++ b/hack/e2e-apps/bucket.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats + +@test "Create and Verify Seeweedfs Bucket" { + # Create the bucket resource + name='test' + kubectl apply -f - < bucket-test-credentials.json + + # Get credentials from the secret + ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' bucket-test-credentials.json) + SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' bucket-test-credentials.json) + BUCKET_NAME=$(jq -r '.spec.bucketName' bucket-test-credentials.json) + + # Start port-forwarding + bash -c 'timeout 100s kubectl port-forward service/seaweedfs-s3 -n tenant-root 8333:8333 > /dev/null 2>&1 &' + + # Wait for port-forward to be ready + timeout 30 sh -ec 'until nc -z localhost 8333; do sleep 1; done' + + # Set up MinIO alias with error handling + mc alias set local https://localhost:8333 $ACCESS_KEY $SECRET_KEY --insecure + + # Upload file to bucket + mc cp bucket-test-credentials.json $BUCKET_NAME/bucket-test-credentials.json + + # Verify file was uploaded + mc ls $BUCKET_NAME/bucket-test-credentials.json + + # Clean up uploaded file + mc rm $BUCKET_NAME/bucket-test-credentials.json + + kubectl -n tenant-test delete bucket.apps.cozystack.io ${name} +} diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 80a0b2d0..d55324b3 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -123,10 +123,10 @@ EOF @test "Configure Tenant and wait for applications" { # Patch root tenant and wait for its releases - kubectl patch tenants/root -n tenant-root --type merge -p '{"spec":{"host":"example.org","ingress":true,"monitoring":true,"etcd":true,"isolated":true}}' + kubectl patch tenants/root -n tenant-root --type merge -p '{"spec":{"host":"example.org","ingress":true,"monitoring":true,"etcd":true,"isolated":true, "seaweedfs": true}}' - timeout 60 sh -ec 'until kubectl get hr -n tenant-root etcd ingress monitoring tenant-root >/dev/null 2>&1; do sleep 1; done' - kubectl wait hr/etcd hr/ingress hr/tenant-root -n tenant-root --timeout=2m --for=condition=ready + timeout 60 sh -ec 'until kubectl get hr -n tenant-root etcd ingress monitoring seaweedfs tenant-root >/dev/null 2>&1; do sleep 1; done' + kubectl wait hr/etcd hr/ingress hr/tenant-root hr/seaweedfs -n tenant-root --timeout=4m --for=condition=ready if ! kubectl wait hr/monitoring -n tenant-root --timeout=2m --for=condition=ready; then flux reconcile hr monitoring -n tenant-root --force diff --git a/packages/core/testing/images/e2e-sandbox/Dockerfile b/packages/core/testing/images/e2e-sandbox/Dockerfile index d67dec13..069a1905 100755 --- a/packages/core/testing/images/e2e-sandbox/Dockerfile +++ b/packages/core/testing/images/e2e-sandbox/Dockerfile @@ -19,6 +19,7 @@ RUN curl -sSL "https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_${TA && chmod +x /usr/local/bin/yq RUN curl -sSL "https://fluxcd.io/install.sh" | bash RUN curl -sSL "https://github.com/cozystack/cozypkg/raw/refs/heads/main/hack/install.sh" | sh -s -- -v "${COZYPKG_VERSION}" - +RUN curl https://dl.min.io/client/mc/release/${TARGETOS}-${TARGETARCH}/mc --create-dirs -o /usr/local/bin/mc \ +&& chmod +x /usr/local/bin/mc COPY entrypoint.sh /usr/local/bin/entrypoint.sh ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] From fa4fff2292c4b79a92db5cd654a3c6bf590252a6 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 13:39:05 +0300 Subject: [PATCH 02/30] [monitoring] add seaweedfs monitoring Signed-off-by: IvanHunters --- dashboards/seaweedfs/seaweedfs.json | 3359 +++++++++++++++++ hack/download-dashboards.sh | 1 + packages/extra/monitoring/Chart.yaml | 2 +- packages/extra/monitoring/dashboards.list | 3 +- packages/extra/seaweedfs/Chart.yaml | 2 +- .../templates/volume-servicemonitor.yaml | 4 +- packages/system/seaweedfs/values.yaml | 8 +- 7 files changed, 3372 insertions(+), 7 deletions(-) create mode 100644 dashboards/seaweedfs/seaweedfs.json diff --git a/dashboards/seaweedfs/seaweedfs.json b/dashboards/seaweedfs/seaweedfs.json new file mode 100644 index 00000000..30b43f86 --- /dev/null +++ b/dashboards/seaweedfs/seaweedfs.json @@ -0,0 +1,3359 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 10423, + "graphTooltip": 0, + "id": 160, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 67, + "panels": [], + "title": "Master", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Whether master is leader or not", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_yes_no", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 57, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "sum by (pod) (SeaweedFS_master_is_leader{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 60 + } + ], + "title": "Raft leader", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Count times leader changed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 68, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "sum by (pod) (SeaweedFS_master_leader_changes{job=\"seaweedfs-master\", type=~\".+\", namespace=\"$NAMESPACE\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 60 + } + ], + "title": "Master leader changes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heartbeats received from components", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 69, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "sum by (type) (increase(SeaweedFS_master_received_heartbeats{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 60 + } + ], + "title": "Received heartbeats", + "type": "timeseries" + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ + { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "message": "", + "name": "Replica Placement Mismatch alert", + "noDataState": "ok", + "notifications": [] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Count replica placement mismatch", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 70, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum (SeaweedFS_master_replica_placement_mismatch{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"} > 0) by (pod)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "op": "gt", + "value": 0, + "visible": true + } + ], + "title": "Replica Placement Mismatch", + "type": "timeseries" + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ + { + "evaluator": { + "params": [ + 1, + 1 + ], + "type": "outside_range" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "message": "Raft leader count of master-servers not equal to 1", + "name": "Raft leader alert", + "noDataState": "no_data", + "notifications": [] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total count of raft leaders", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 71, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "sum (SeaweedFS_master_is_leader{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Leaders", + "refId": "A", + "step": 60 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "op": "lt", + "value": 1, + "visible": true + }, + { + "colorMode": "critical", + "op": "gt", + "value": 1, + "visible": true + } + ], + "title": "Raft leader count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Whether cluster locked or not", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_yes_no", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 74, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "SeaweedFS_master_admin_lock{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Client IP: {{client}}", + "range": true, + "refId": "A", + "step": 60 + } + ], + "title": "Admin lock", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 60, + "panels": [], + "title": "Filer", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 46, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "title": "Filer Request Duration 90th percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 49, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 2, + "refId": "C" + } + ], + "title": "Filer Request Duration 95th percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 45, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 2, + "refId": "C" + } + ], + "title": "Filer Request Duration 99th percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 2, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "sum by (type) (rate(SeaweedFS_filer_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "title": "Filer QPS", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 61, + "panels": [], + "title": "S3 Gateway", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 29 + }, + "id": 65, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "title": "S3 Request Duration 90th percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 29 + }, + "id": 56, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "title": "S3 Request Duration 95th percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 29 + }, + "id": 58, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "title": "S3 Request Duration 99th percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 84, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "sum(rate(SeaweedFS_s3_bucket_traffic_received_bytes_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "S3 Bucket Traffic Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 85, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "sum(rate(SeaweedFS_s3_bucket_traffic_sent_bytes_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "S3 Bucket Traffic Sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 72, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type, pod))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}-{{pod}}", + "refId": "B", + "step": 60 + } + ], + "title": "S3 Request. Duration 99th percentile per instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 73, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type, bucket))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}-{{bucket}}", + "refId": "B", + "step": 60 + } + ], + "title": "S3 Request Duration 99th percentile per bucket", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 55, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum (rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "title": "S3 API QPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Cost in US$", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 64 + }, + "hideTimeOverride": false, + "id": 59, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "sum by (type) (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST', namespace=\"$NAMESPACE\"})*0.000005", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}} requests", + "refId": "A", + "step": 30 + }, + { + "expr": "sum (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST', namespace=\"$NAMESPACE\"})*0.000005", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "All PUT, COPY, POST, LIST", + "refId": "C", + "step": 30 + }, + { + "expr": "sum (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST', namespace=\"$NAMESPACE\"})*0.0000004", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "GET and all other", + "refId": "B" + }, + { + "expr": "sum by (type) (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST', namespace=\"$NAMESPACE\"})*0.0000004", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}} requests", + "refId": "D" + } + ], + "timeFrom": "1M", + "title": "S3 API Monthly Cost if on AWS", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 62, + "panels": [], + "title": "Volume Server", + "type": "row" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 47, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, exported_instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "C" + } + ], + "title": "Volume Server Request Duration 99th percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 40, + "links": [], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "sum(rate(SeaweedFS_volumeServer_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 4 + } + ], + "title": "Volume Server QPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 48, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(SeaweedFS_volumeServer_volumes{namespace=\"$NAMESPACE\"}) by (collection, type)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{collection}} {{type}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(SeaweedFS_volumeServer_volumes{namespace=\"$NAMESPACE\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total", + "refId": "B" + } + ], + "title": "Volume Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 50, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"$NAMESPACE\"}) by (collection, type)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{collection}} {{type}}", + "refId": "A" + }, + { + "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"$NAMESPACE\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total", + "refId": "B" + } + ], + "title": "Used Disk Space by Collection and Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 51, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"$NAMESPACE\"}) by (exported_instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "A" + } + ], + "title": "Used Disk Space by Host", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 63, + "panels": [], + "title": "Filer Store", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 101 + }, + "id": 12, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B" + } + ], + "title": "Filer Store Request Duration 99th percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 101 + }, + "id": 14, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "expr": "sum(rate(SeaweedFS_filerStore_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B" + } + ], + "title": "Filer Store QPS", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 108 + }, + "id": 64, + "panels": [], + "title": "Filer Instances", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 109 + }, + "id": 52, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_alloc_bytes{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "bytes allocated", + "refId": "B" + }, + { + "exemplar": true, + "expr": "rate(go_memstats_alloc_bytes_total{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "alloc rate", + "refId": "A" + }, + { + "exemplar": true, + "expr": "go_memstats_stack_inuse_bytes{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "stack inuse", + "refId": "C" + }, + { + "exemplar": true, + "expr": "go_memstats_heap_inuse_bytes{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "heap inuse", + "refId": "D" + } + ], + "title": "Filer Go Memory Stats", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 109 + }, + "id": 54, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "go_gc_duration_seconds{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{quantile}}", + "refId": "B" + } + ], + "title": "Filer Go GC duration quantiles", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 116 + }, + "id": 53, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "go_goroutines{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "B" + } + ], + "title": "Filer Go Routines", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "mes", + "value": "mes" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(SeaweedFS_master_is_leader,namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "NAMESPACE", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(SeaweedFS_master_is_leader,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "SeaweedFS", + "uid": "a24009d7-cbda-4443-a132-1cc1c4677304", + "version": 1, + "weekStart": "" +} diff --git a/hack/download-dashboards.sh b/hack/download-dashboards.sh index ade7ca0b..a1a6082a 100755 --- a/hack/download-dashboards.sh +++ b/hack/download-dashboards.sh @@ -81,6 +81,7 @@ modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//main/capacity-p modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//flux/flux-control-plane.json modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//flux/flux-stats.json modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//kafka/strimzi-kafka.json +modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//seaweedfs/seaweedfs.json modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//goldpinger/goldpinger.json EOT diff --git a/packages/extra/monitoring/Chart.yaml b/packages/extra/monitoring/Chart.yaml index ffc79ede..75a6ab12 100644 --- a/packages/extra/monitoring/Chart.yaml +++ b/packages/extra/monitoring/Chart.yaml @@ -3,4 +3,4 @@ name: monitoring description: Monitoring and observability stack icon: /logos/monitoring.svg type: application -version: 1.12.0 +version: 1.12.1 diff --git a/packages/extra/monitoring/dashboards.list b/packages/extra/monitoring/dashboards.list index dec05cf5..21ecb974 100644 --- a/packages/extra/monitoring/dashboards.list +++ b/packages/extra/monitoring/dashboards.list @@ -37,4 +37,5 @@ flux/flux-stats kafka/strimzi-kafka goldpinger/goldpinger clickhouse/altinity-clickhouse-operator-dashboard -storage/linstor \ No newline at end of file +storage/linstor +seaweedfs/seaweedfs diff --git a/packages/extra/seaweedfs/Chart.yaml b/packages/extra/seaweedfs/Chart.yaml index 24796add..2e8e0606 100644 --- a/packages/extra/seaweedfs/Chart.yaml +++ b/packages/extra/seaweedfs/Chart.yaml @@ -22,4 +22,4 @@ version: 0.6.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "3.71" +appVersion: "3.72" diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml index dd8a9f9d..b60e9e44 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml @@ -21,9 +21,9 @@ metadata: {{- with $.Values.global.monitoring.additionalLabels }} {{- toYaml . | nindent 4 }} {{- end }} -{{- if .Values.volume.annotations }} +{{- if $.Values.volume.annotations }} annotations: - {{- toYaml .Values.volume.annotations | nindent 4 }} + {{- toYaml $.Values.volume.annotations | nindent 4 }} {{- end }} spec: endpoints: diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 8aaeea2b..e2f625f0 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -4,6 +4,8 @@ global: extraEnvironmentVars: WEED_CLUSTER_SW_MASTER: "seaweedfs-master:9333" WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client:8888" + monitoring: + enabled: true seaweedfs: master: replicas: 3 @@ -67,8 +69,10 @@ seaweedfs: name: seaweedfs-db-app s3: enabled: true - port: 8333 - httpsPort: 0 + metricsPort: 9334 + enableAuth: true + readinessProbe: + scheme: HTTPS # Suffix of the host name, {bucket}.{domainName} domainName: "" # enable user & permission to s3 (need to inject to all services) From 0e428810fd9876e454b3b00cb7a14e5da35e65f4 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 16:02:12 +0300 Subject: [PATCH 03/30] [fix] add patches Signed-off-by: IvanHunters --- packages/extra/monitoring/Chart.yaml | 2 +- .../patches/add-seaweedfs-dashboard.patch | 11 +++++++++++ packages/extra/seaweedfs/Chart.yaml | 4 ++-- .../patches/fix-volume-servicemonitor.patch | 15 +++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 packages/extra/monitoring/patches/add-seaweedfs-dashboard.patch create mode 100644 packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch diff --git a/packages/extra/monitoring/Chart.yaml b/packages/extra/monitoring/Chart.yaml index 75a6ab12..9ee3a2bb 100644 --- a/packages/extra/monitoring/Chart.yaml +++ b/packages/extra/monitoring/Chart.yaml @@ -3,4 +3,4 @@ name: monitoring description: Monitoring and observability stack icon: /logos/monitoring.svg type: application -version: 1.12.1 +version: 1.12.2 diff --git a/packages/extra/monitoring/patches/add-seaweedfs-dashboard.patch b/packages/extra/monitoring/patches/add-seaweedfs-dashboard.patch new file mode 100644 index 00000000..e504d085 --- /dev/null +++ b/packages/extra/monitoring/patches/add-seaweedfs-dashboard.patch @@ -0,0 +1,11 @@ +diff --git a/packages/extra/monitoring/dashboards.list b/packages/extra/monitoring/dashboards.list +--- a/packages/extra/monitoring/dashboards.list (revision 8951bc13d7d02b5e6982a239570ed58ed7cb025a) ++++ b/packages/extra/monitoring/dashboards.list (revision fa4fff2292c4b79a92db5cd654a3c6bf590252a6) +@@ -37,4 +37,5 @@ + kafka/strimzi-kafka + goldpinger/goldpinger + clickhouse/altinity-clickhouse-operator-dashboard +-storage/linstor +\ No newline at end of file ++storage/linstor ++seaweedfs/seaweedfs \ No newline at end of file diff --git a/packages/extra/seaweedfs/Chart.yaml b/packages/extra/seaweedfs/Chart.yaml index 2e8e0606..4b5c8122 100644 --- a/packages/extra/seaweedfs/Chart.yaml +++ b/packages/extra/seaweedfs/Chart.yaml @@ -16,10 +16,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.6.0 +version: 0.6.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "3.72" +appVersion: "3.71" diff --git a/packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch b/packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch new file mode 100644 index 00000000..b6b40527 --- /dev/null +++ b/packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch @@ -0,0 +1,15 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml (revision 8951bc13d7d02b5e6982a239570ed58ed7cb025a) ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml (revision fa4fff2292c4b79a92db5cd654a3c6bf590252a6) +@@ -21,9 +21,9 @@ + {{- with $.Values.global.monitoring.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +-{{- if .Values.volume.annotations }} ++{{- if $.Values.volume.annotations }} + annotations: +- {{- toYaml .Values.volume.annotations | nindent 4 }} ++ {{- toYaml $.Values.volume.annotations | nindent 4 }} + {{- end }} + spec: + endpoints: \ No newline at end of file From 7918e282bf42c08f40f966089535f1acc931f93a Mon Sep 17 00:00:00 2001 From: kklinch0 Date: Wed, 30 Jul 2025 16:07:22 +0300 Subject: [PATCH 04/30] keycloak enable cookie-secure Signed-off-by: kklinch0 --- .../system/keycloak-configure/templates/configure-kk.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/system/keycloak-configure/templates/configure-kk.yaml b/packages/system/keycloak-configure/templates/configure-kk.yaml index adee11b6..00673c03 100644 --- a/packages/system/keycloak-configure/templates/configure-kk.yaml +++ b/packages/system/keycloak-configure/templates/configure-kk.yaml @@ -200,7 +200,7 @@ spec: - groups - kubernetes-client redirectUris: - - "http://dashboard.{{ $host }}/oauth2/callback/*" + - "https://dashboard.{{ $host }}/oauth2/callback/*" {{- range $i, $v := $extraRedirectUris }} - "{{ $v }}" {{- end }} @@ -223,8 +223,6 @@ data: clientSecret: {{ $kubeappsClient }} cookieSecret: {{ $cookieSecret }} extraFlags: - - --ssl-insecure-skip-verify - - --cookie-secure=false - --scope=openid email groups - --oidc-issuer-url=https://keycloak.{{ $host }}/realms/cozy From e3a61b23afb5333aed53868d4e2142b86177de46 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 16:20:09 +0300 Subject: [PATCH 05/30] fix versions map Signed-off-by: IvanHunters --- packages/extra/versions_map | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/extra/versions_map b/packages/extra/versions_map index bc581f82..b86aa1fc 100644 --- a/packages/extra/versions_map +++ b/packages/extra/versions_map @@ -47,7 +47,8 @@ monitoring 1.9.2 f9f8bb2f monitoring 1.10.0 632224a3 monitoring 1.10.1 8c86905b monitoring 1.11.0 4369b031 -monitoring 1.12.0 HEAD +monitoring 1.12.0 0e47e1e8 +monitoring 1.12.2 HEAD seaweedfs 0.1.0 71514249 seaweedfs 0.2.0 5fb9cfe3 seaweedfs 0.2.1 fde4bcfa @@ -55,4 +56,5 @@ seaweedfs 0.3.0 45a7416c seaweedfs 0.4.0 632224a3 seaweedfs 0.4.1 8c86905b seaweedfs 0.5.0 9584e5f5 -seaweedfs 0.6.0 HEAD +seaweedfs 0.6.0 0e47e1e8 +seaweedfs 0.6.1 HEAD From 2897813ddaee8b161755e341b5d5d797ef6eb231 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 16:26:01 +0300 Subject: [PATCH 06/30] revert bump version of seaweed in extra Signed-off-by: IvanHunters --- packages/extra/seaweedfs/Chart.yaml | 2 +- packages/extra/versions_map | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/extra/seaweedfs/Chart.yaml b/packages/extra/seaweedfs/Chart.yaml index 4b5c8122..24796add 100644 --- a/packages/extra/seaweedfs/Chart.yaml +++ b/packages/extra/seaweedfs/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.6.1 +version: 0.6.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/packages/extra/versions_map b/packages/extra/versions_map index b86aa1fc..7883f603 100644 --- a/packages/extra/versions_map +++ b/packages/extra/versions_map @@ -56,5 +56,4 @@ seaweedfs 0.3.0 45a7416c seaweedfs 0.4.0 632224a3 seaweedfs 0.4.1 8c86905b seaweedfs 0.5.0 9584e5f5 -seaweedfs 0.6.0 0e47e1e8 -seaweedfs 0.6.1 HEAD +seaweedfs 0.6.0 HEAD From 0d7e85618641684b25d98d1b31c0316a78126028 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 16:29:44 +0300 Subject: [PATCH 07/30] delete patch for seaweedfs Signed-off-by: IvanHunters --- .../patches/fix-volume-servicemonitor.patch | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch diff --git a/packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch b/packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch deleted file mode 100644 index b6b40527..00000000 --- a/packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml ---- a/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml (revision 8951bc13d7d02b5e6982a239570ed58ed7cb025a) -+++ b/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml (revision fa4fff2292c4b79a92db5cd654a3c6bf590252a6) -@@ -21,9 +21,9 @@ - {{- with $.Values.global.monitoring.additionalLabels }} - {{- toYaml . | nindent 4 }} - {{- end }} --{{- if .Values.volume.annotations }} -+{{- if $.Values.volume.annotations }} - annotations: -- {{- toYaml .Values.volume.annotations | nindent 4 }} -+ {{- toYaml $.Values.volume.annotations | nindent 4 }} - {{- end }} - spec: - endpoints: \ No newline at end of file From 824c72318aec8ccdccf12e760c68363776f97d01 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 16:31:24 +0300 Subject: [PATCH 08/30] fix patches Signed-off-by: IvanHunters --- .../patches/add-seaweedfs-dashboard.patch | 11 ----------- .../patches/fix-volume-servicemonitor.patch | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) delete mode 100644 packages/extra/monitoring/patches/add-seaweedfs-dashboard.patch create mode 100644 packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch diff --git a/packages/extra/monitoring/patches/add-seaweedfs-dashboard.patch b/packages/extra/monitoring/patches/add-seaweedfs-dashboard.patch deleted file mode 100644 index e504d085..00000000 --- a/packages/extra/monitoring/patches/add-seaweedfs-dashboard.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff --git a/packages/extra/monitoring/dashboards.list b/packages/extra/monitoring/dashboards.list ---- a/packages/extra/monitoring/dashboards.list (revision 8951bc13d7d02b5e6982a239570ed58ed7cb025a) -+++ b/packages/extra/monitoring/dashboards.list (revision fa4fff2292c4b79a92db5cd654a3c6bf590252a6) -@@ -37,4 +37,5 @@ - kafka/strimzi-kafka - goldpinger/goldpinger - clickhouse/altinity-clickhouse-operator-dashboard --storage/linstor -\ No newline at end of file -+storage/linstor -+seaweedfs/seaweedfs \ No newline at end of file diff --git a/packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch b/packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch new file mode 100644 index 00000000..b6b40527 --- /dev/null +++ b/packages/system/seaweedfs/patches/fix-volume-servicemonitor.patch @@ -0,0 +1,15 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml (revision 8951bc13d7d02b5e6982a239570ed58ed7cb025a) ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/volume-servicemonitor.yaml (revision fa4fff2292c4b79a92db5cd654a3c6bf590252a6) +@@ -21,9 +21,9 @@ + {{- with $.Values.global.monitoring.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +-{{- if .Values.volume.annotations }} ++{{- if $.Values.volume.annotations }} + annotations: +- {{- toYaml .Values.volume.annotations | nindent 4 }} ++ {{- toYaml $.Values.volume.annotations | nindent 4 }} + {{- end }} + spec: + endpoints: \ No newline at end of file From 940b0b18b0ab1e4be73dc327b8da91618aa16ec5 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 16:39:14 +0300 Subject: [PATCH 09/30] fix values for seadeed monitoring Signed-off-by: IvanHunters --- packages/system/seaweedfs/values.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index e2f625f0..8510f2ef 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -69,10 +69,8 @@ seaweedfs: name: seaweedfs-db-app s3: enabled: true - metricsPort: 9334 - enableAuth: true - readinessProbe: - scheme: HTTPS + port: 8333 + httpsPort: 0 # Suffix of the host name, {bucket}.{domainName} domainName: "" # enable user & permission to s3 (need to inject to all services) @@ -83,6 +81,9 @@ seaweedfs: auditLogConfig: {} s3: enableAuth: true + metricsPort: 9334 + readinessProbe: + scheme: HTTPS logs: type: "" ingress: From 53241efe63e219fb112f2382577c06b696bd8895 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 16:46:27 +0300 Subject: [PATCH 10/30] fix values file Signed-off-by: IvanHunters --- packages/system/seaweedfs/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 8510f2ef..2a3341e2 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -80,6 +80,7 @@ seaweedfs: existingConfigSecret: null auditLogConfig: {} s3: + enabled: true enableAuth: true metricsPort: 9334 readinessProbe: From 1d9465d662455dfc40184a017e06d9ee17c4ff59 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 16:59:11 +0300 Subject: [PATCH 11/30] revert of metrics port for s3 Signed-off-by: IvanHunters --- packages/system/seaweedfs/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 2a3341e2..0c0cad7f 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -82,7 +82,6 @@ seaweedfs: s3: enabled: true enableAuth: true - metricsPort: 9334 readinessProbe: scheme: HTTPS logs: From dca2eb7ae829de4f2edc8f35db4608974a4834b1 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 17:08:54 +0300 Subject: [PATCH 12/30] fix chart version Signed-off-by: IvanHunters --- packages/extra/monitoring/Chart.yaml | 2 +- packages/extra/versions_map | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/extra/monitoring/Chart.yaml b/packages/extra/monitoring/Chart.yaml index 9ee3a2bb..75a6ab12 100644 --- a/packages/extra/monitoring/Chart.yaml +++ b/packages/extra/monitoring/Chart.yaml @@ -3,4 +3,4 @@ name: monitoring description: Monitoring and observability stack icon: /logos/monitoring.svg type: application -version: 1.12.2 +version: 1.12.1 diff --git a/packages/extra/versions_map b/packages/extra/versions_map index 7883f603..232486b4 100644 --- a/packages/extra/versions_map +++ b/packages/extra/versions_map @@ -48,7 +48,7 @@ monitoring 1.10.0 632224a3 monitoring 1.10.1 8c86905b monitoring 1.11.0 4369b031 monitoring 1.12.0 0e47e1e8 -monitoring 1.12.2 HEAD +monitoring 1.12.1 HEAD seaweedfs 0.1.0 71514249 seaweedfs 0.2.0 5fb9cfe3 seaweedfs 0.2.1 fde4bcfa From 03c4bf904f2ccf6c8b38a37669c6b3788c431f2b Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 30 Jul 2025 17:38:50 +0300 Subject: [PATCH 13/30] add handle of patch in makefile Signed-off-by: IvanHunters --- packages/system/seaweedfs/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/system/seaweedfs/Makefile b/packages/system/seaweedfs/Makefile index 2d51079b..c9fd7b74 100644 --- a/packages/system/seaweedfs/Makefile +++ b/packages/system/seaweedfs/Makefile @@ -8,4 +8,5 @@ update: curl -sSL https://github.com/seaweedfs/seaweedfs/archive/refs/heads/master.tar.gz | \ tar xzvf - --strip 3 -C charts seaweedfs-master/k8s/charts/seaweedfs patch --no-backup-if-mismatch -p4 < patches/resize-api-server-annotation.diff + patch --no-backup-if-mismatch -p4 < patches/fix-volume-servicemonitor.patch #patch --no-backup-if-mismatch -p4 < patches/retention-policy-delete.yaml From 8704767ac5e58c704ced76a25f49ccdbbf414b0b Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 30 Jul 2025 16:44:34 +0200 Subject: [PATCH 14/30] [dx] Allow to not specify BUILDER for makefile if PLATFORM specified Signed-off-by: Andrei Kvapil --- scripts/common-envs.mk | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/common-envs.mk b/scripts/common-envs.mk index bdff841b..ec11ada4 100644 --- a/scripts/common-envs.mk +++ b/scripts/common-envs.mk @@ -17,6 +17,8 @@ endif # Get the name of the selected docker buildx builder BUILDER ?= $(shell docker buildx inspect --bootstrap | head -n2 | awk '/^Name:/{print $$NF}') -# Get platforms supported by the builder -PLATFORM ?= $(shell docker buildx ls --format=json | jq -r 'select(.Name == "$(BUILDER)") | [.Nodes[].Platforms // []] | flatten | unique | map(select(test("^linux/amd64$$|^linux/arm64$$"))) | join(",")') +# Get platforms supported by the builder (only if PLATFORM is not provided) +ifeq ($(origin PLATFORM), undefined) +PLATFORM := $(shell docker buildx ls --format=json | jq -r 'select(.Name == "$(BUILDER)") | [.Nodes[].Platforms // []] | flatten | unique | map(select(test("^linux/amd64$$|^linux/arm64$$"))) | join(",")') +endif From 36ccfb9509295ee9aa69c639af63e8dfaa59bdab Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Thu, 31 Jul 2025 11:30:35 +0300 Subject: [PATCH 15/30] add limits for s3 deployment Signed-off-by: IvanHunters --- packages/system/seaweedfs/values.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 0c0cad7f..8ffeb6b0 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -99,6 +99,13 @@ seaweedfs: - hosts: - seaweedfs.demo.cozystack.io secretName: seaweedfs-s3-ingress-tls + resources: + limits: + cpu: "2" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" cosi: enabled: true podLabels: From 1ab63187c97d65134dcd9ccca208730260642490 Mon Sep 17 00:00:00 2001 From: klinch0 <68821526+klinch0@users.noreply.github.com> Date: Thu, 31 Jul 2025 17:13:25 +0300 Subject: [PATCH 16/30] Update packages/system/keycloak-configure/templates/configure-kk.yaml Co-authored-by: Timofei Larkin Signed-off-by: klinch0 <68821526+klinch0@users.noreply.github.com> --- packages/system/keycloak-configure/templates/configure-kk.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/system/keycloak-configure/templates/configure-kk.yaml b/packages/system/keycloak-configure/templates/configure-kk.yaml index 00673c03..c49a3889 100644 --- a/packages/system/keycloak-configure/templates/configure-kk.yaml +++ b/packages/system/keycloak-configure/templates/configure-kk.yaml @@ -223,6 +223,7 @@ data: clientSecret: {{ $kubeappsClient }} cookieSecret: {{ $cookieSecret }} extraFlags: + - --cookie-secure - --scope=openid email groups - --oidc-issuer-url=https://keycloak.{{ $host }}/realms/cozy From f057d92a4d66fd11b3eb35ba16d95e9a9d96ee3d Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Thu, 31 Jul 2025 17:18:25 +0200 Subject: [PATCH 17/30] [cozystack-api] fix type for ApplicationList Signed-off-by: Andrei Kvapil --- pkg/registry/apps/application/rest.go | 46 +++++++++++++++++---------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index cdca3329..27a319a8 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -271,7 +271,6 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption klog.Errorf("Invalid field selector: %v", err) return nil, fmt.Errorf("invalid field selector: %v", err) } - // Check if selector is for metadata.name if name, exists := fs.RequiresExactMatch("metadata.name"); exists { // Convert Application name to HelmRelease name @@ -321,17 +320,8 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption return nil, err } - // Initialize empty Application list - appList := &appsv1alpha1.ApplicationList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps.cozystack.io/v1alpha1", - Kind: "ApplicationList", - }, - ListMeta: metav1.ListMeta{ - ResourceVersion: hrList.GetResourceVersion(), - }, - Items: []appsv1alpha1.Application{}, - } + // Initialize unstructured items array + items := make([]unstructured.Unstructured, 0) // Iterate over HelmReleases and convert to Applications for _, hr := range hrList.Items { @@ -369,7 +359,6 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption klog.Errorf("Invalid field selector: %v", err) continue } - fieldsSet := fields.Set{ "metadata.name": app.Name, "metadata.namespace": app.Namespace, @@ -379,10 +368,23 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption } } - appList.Items = append(appList.Items, app) + // Convert Application to unstructured + unstructuredApp, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&app) + if err != nil { + klog.Errorf("Error converting Application %s to unstructured: %v", app.Name, err) + continue + } + items = append(items, unstructured.Unstructured{Object: unstructuredApp}) } - klog.V(6).Infof("Successfully listed %d Application resources in namespace %s", len(appList.Items), namespace) + // Explicitly set apiVersion and kind in unstructured object + appList := &unstructured.UnstructuredList{} + appList.SetAPIVersion("apps.cozystack.io/v1alpha1") + appList.SetKind(r.kindName + "List") + appList.SetResourceVersion(hrList.GetResourceVersion()) + appList.Items = items + + klog.V(6).Infof("Successfully listed %d Application resources in namespace %s", len(items), namespace) return appList, nil } @@ -1036,6 +1038,19 @@ func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableO case *appsv1alpha1.Application: table = r.buildTableFromApplication(*obj) table.ListMeta.ResourceVersion = obj.GetResourceVersion() + case *unstructured.UnstructuredList: + apps := make([]appsv1alpha1.Application, 0, len(obj.Items)) + for _, u := range obj.Items { + var a appsv1alpha1.Application + err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &a) + if err != nil { + klog.Errorf("Failed to convert Unstructured to Application: %v", err) + continue + } + apps = append(apps, a) + } + table = r.buildTableFromApplications(apps) + table.ListMeta.ResourceVersion = obj.GetResourceVersion() case *unstructured.Unstructured: var app appsv1alpha1.Application err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &app) @@ -1067,7 +1082,6 @@ func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableO } klog.V(6).Infof("ConvertToTable: returning table with %d rows", len(table.Rows)) - return &table, nil } From 1c2cc0fa289cba58a711a6ae8e9a194cf5f2ca18 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 1 Aug 2025 15:54:09 +0300 Subject: [PATCH 18/30] [monitoring] more retries Signed-off-by: IvanHunters --- packages/apps/tenant/templates/monitoring.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/apps/tenant/templates/monitoring.yaml b/packages/apps/tenant/templates/monitoring.yaml index 1992a4ea..e1f9fcf3 100644 --- a/packages/apps/tenant/templates/monitoring.yaml +++ b/packages/apps/tenant/templates/monitoring.yaml @@ -17,6 +17,12 @@ spec: kind: HelmRepository name: cozystack-extra namespace: cozy-public + install: + remediation: + retries: 10 + upgrade: + remediation: + retries: 10 interval: 1m0s - timeout: 5m0s + timeout: 10m0s {{- end }} From 9d184a098f31fd4cbbaf9816f167f6f0533105b3 Mon Sep 17 00:00:00 2001 From: cozystack-bot <217169706+cozystack-bot@users.noreply.github.com> Date: Fri, 1 Aug 2025 13:07:01 +0000 Subject: [PATCH 19/30] Prepare release v0.35.0-alpha.3 Signed-off-by: cozystack-bot <217169706+cozystack-bot@users.noreply.github.com> --- packages/apps/http-cache/images/nginx-cache.tag | 2 +- packages/apps/kubernetes/images/cluster-autoscaler.tag | 2 +- .../apps/kubernetes/images/kubevirt-cloud-provider.tag | 2 +- packages/apps/kubernetes/images/kubevirt-csi-driver.tag | 2 +- packages/core/installer/values.yaml | 2 +- packages/core/testing/values.yaml | 2 +- packages/extra/bootbox/images/matchbox.tag | 2 +- packages/extra/monitoring/images/grafana.tag | 2 +- packages/extra/seaweedfs/images/objectstorage-sidecar.tag | 2 +- packages/system/cozystack-api/values.yaml | 2 +- packages/system/cozystack-controller/values.yaml | 4 ++-- .../charts/kubeapps/templates/dashboard/configmap.yaml | 2 +- packages/system/dashboard/values.yaml | 8 ++++---- packages/system/kamaji/values.yaml | 2 +- packages/system/kubeovn-webhook/values.yaml | 2 +- packages/system/kubeovn/values.yaml | 2 +- packages/system/kubevirt-csi-node/values.yaml | 2 +- packages/system/objectstorage-controller/values.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- 19 files changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index 987b0886..5452b4cf 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:0.6.1@sha256:50ac1581e3100bd6c477a71161cb455a341ffaf9e5e2f6086802e4e25271e8af +ghcr.io/cozystack/cozystack/nginx-cache:0.6.1@sha256:b7633717cd7449c0042ae92d8ca9b36e4d69566561f5c7d44e21058e7d05c6d5 diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index 112eac53..7e710659 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:0.26.1@sha256:3a8170433e1632e5cc2b6d9db34d0605e8e6c63c158282c38450415e700e932e +ghcr.io/cozystack/cozystack/cluster-autoscaler:0.26.2@sha256:3a8170433e1632e5cc2b6d9db34d0605e8e6c63c158282c38450415e700e932e diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index ec6955a4..3535de68 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:0.26.1@sha256:5335c044313b69ee13b30ca4941687e509005e55f4ae25723861edbf2fbd6dd2 +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:0.26.2@sha256:5335c044313b69ee13b30ca4941687e509005e55f4ae25723861edbf2fbd6dd2 diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index 27f0c7e6..e78ff813 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.26.1@sha256:445c2727b04ac68595b43c988ff17b3d69a7b22b0644fde3b10c65b47a7bc036 +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.26.2@sha256:761e7235ff9cb7f6f223f00954943e6a5af32ed6624ee592a8610122f96febb0 diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index 50c0f88f..f8115dd2 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -1,2 +1,2 @@ cozystack: - image: ghcr.io/cozystack/cozystack/installer:v0.35.0-alpha.2@sha256:ecdda158d9f8e1b83fb75e129c33abb0763f3ba32920e9e9192e1ac6946f4048 + image: ghcr.io/cozystack/cozystack/installer:v0.35.0-alpha.3@sha256:b43615350697dbbc377d31152b290722c2bba7a053fd7c69403d666e6c7d53ee diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index 84c13ba5..3188c87b 100755 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v0.35.0-alpha.2@sha256:dd0b7d231ad8d332b00ef69000ffa570d1d96c6443a2dd9a0b31678657f624f5 + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v0.35.0-alpha.3@sha256:cd6d65230150171c0e3934f71fcc270718f2bfec7509989b9d4d996ddef916d9 diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index c9957a6f..f8371c83 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v0.35.0-alpha.2@sha256:61684ddd5ca56c6031176e4b6aef6c979dbe54ad873172dda3a25a63aca22dbb +ghcr.io/cozystack/cozystack/matchbox:v0.35.0-alpha.3@sha256:043051d7c368b73943051b51bacfb3e383541b53190f74452d552ea2f30e420e diff --git a/packages/extra/monitoring/images/grafana.tag b/packages/extra/monitoring/images/grafana.tag index 5bc2f99a..715b03a1 100644 --- a/packages/extra/monitoring/images/grafana.tag +++ b/packages/extra/monitoring/images/grafana.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana:1.12.0@sha256:c63978e1ed0304e8518b31ddee56c4e8115541b997d8efbe1c0a74da57140399 +ghcr.io/cozystack/cozystack/grafana:1.12.1@sha256:c63978e1ed0304e8518b31ddee56c4e8115541b997d8efbe1c0a74da57140399 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index 720e58b6..c38b73e2 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v0.35.0-alpha.2@sha256:e751c20edbefb1e458d9272ec5faea011e41bb828c067a562d8ddbd8d9da7242 +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v0.35.0-alpha.3@sha256:4a67ec40c13bf912151957dc0e7409f7e01a2859905285c98c20baae7b667ae2 diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index 66296d9d..3a1794b7 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,2 +1,2 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v0.35.0-alpha.2@sha256:677b5af99f2b7d1adb4c25d5637f2fa5a1baffb423269fa166d2ea15a0028864 + image: ghcr.io/cozystack/cozystack/cozystack-api:v0.35.0-alpha.3@sha256:d4cfc2afacc30d383f7b5454e97bb77f2e9bfc33dbc3dc512bfe671300e3ad51 diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index d2c54c80..83df9b48 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,5 +1,5 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v0.35.0-alpha.2@sha256:678c07ca658bbde551ba487537e483615019ea161a849243139399faa73740cf + image: ghcr.io/cozystack/cozystack/cozystack-controller:v0.35.0-alpha.3@sha256:98d0b945630c3dd7c39adc7a5326f0e0e0b49a18641bec911aeb6c62beca4e1f debug: false disableTelemetry: false - cozystackVersion: "v0.35.0-alpha.2" + cozystackVersion: "v0.35.0-alpha.3" diff --git a/packages/system/dashboard/charts/kubeapps/templates/dashboard/configmap.yaml b/packages/system/dashboard/charts/kubeapps/templates/dashboard/configmap.yaml index a1712298..07d6beed 100644 --- a/packages/system/dashboard/charts/kubeapps/templates/dashboard/configmap.yaml +++ b/packages/system/dashboard/charts/kubeapps/templates/dashboard/configmap.yaml @@ -76,7 +76,7 @@ data: "kubeappsNamespace": {{ .Release.Namespace | quote }}, "helmGlobalNamespace": {{ include "kubeapps.helmGlobalPackagingNamespace" . | quote }}, "carvelGlobalNamespace": {{ .Values.kubeappsapis.pluginConfig.kappController.packages.v1alpha1.globalPackagingNamespace | quote }}, - "appVersion": "v0.35.0-alpha.2", + "appVersion": "v0.35.0-alpha.3", "authProxyEnabled": {{ .Values.authProxy.enabled }}, "oauthLoginURI": {{ .Values.authProxy.oauthLoginURI | quote }}, "oauthLogoutURI": {{ .Values.authProxy.oauthLogoutURI | quote }}, diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index ecfbb22e..73f6638d 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -19,8 +19,8 @@ kubeapps: image: registry: ghcr.io/cozystack/cozystack repository: dashboard - tag: v0.35.0-alpha.2 - digest: "sha256:84c333ce0ecddb153a6886b1c633e4480c78690fb0b49cc0f4cb980bee565134" + tag: v0.35.0-alpha.3 + digest: "sha256:ad4b95660b6c5c1b9736ca4768a3f9648705c2855d0a08880d570b4e480dba78" redis: master: resourcesPreset: "none" @@ -37,8 +37,8 @@ kubeapps: image: registry: ghcr.io/cozystack/cozystack repository: kubeapps-apis - tag: v0.35.0-alpha.2 - digest: "sha256:69887cc50059330a8b05c443102e3dedd5cff1543be59066b31f0ad97a727c23" + tag: v0.35.0-alpha.3 + digest: "sha256:55b1aacdcb8f7e96b75396e461fbba612498095515b9a437b8f925802f06c485" pluginConfig: flux: packages: diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 5845b23a..2f5991d9 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -3,7 +3,7 @@ kamaji: deploy: false image: pullPolicy: IfNotPresent - tag: v0.35.0-alpha.2@sha256:dd662e320f44342fe5f10d63545bf1915e97de380c1685e7b28f7a7fce87512e + tag: v0.35.0-alpha.3@sha256:34500654751ceef7a57562fb1edc6928d91245ad8e41da732788dcf10ec330b5 repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index df445ccd..383bda1a 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v0.35.0-alpha.2@sha256:782a84c9afa5dcea2208f71987d019d573a23676a3e9df6df0cfcd639cf4d7a0 +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v0.35.0-alpha.3@sha256:ad4e506065458b039a86c7f03a35dbb6918756e117ce74dd931089383cfd91f8 diff --git a/packages/system/kubeovn/values.yaml b/packages/system/kubeovn/values.yaml index bd8e5c73..c1b1a8cc 100644 --- a/packages/system/kubeovn/values.yaml +++ b/packages/system/kubeovn/values.yaml @@ -64,4 +64,4 @@ global: images: kubeovn: repository: kubeovn - tag: v1.13.14@sha256:5cce383ac38e56b1e3acb2fc29d9ae042afa269035b4f57e6737265e42ea2ef8 + tag: v1.13.14@sha256:bacd68243ee9fd94f5f0e377801ed3e99cd0762ff5df0d0ec73a73130180d8ca diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index ea2980d8..645ef8f1 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -1,3 +1,3 @@ storageClass: replicated csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.26.1@sha256:445c2727b04ac68595b43c988ff17b3d69a7b22b0644fde3b10c65b47a7bc036 + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.26.2@sha256:761e7235ff9cb7f6f223f00954943e6a5af32ed6624ee592a8610122f96febb0 diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index 328a4876..ca303923 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v0.35.0-alpha.2@sha256:f57500d935d89ceb76241bf1257be64fd6624f4881df7a5390ff7149667eb62c" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v0.35.0-alpha.3@sha256:4d3f20ae946736d9260193dd1ce0e72114ad6bd6f78fb6528a68efffa2092b07" diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 8ffeb6b0..26afa71c 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -114,7 +114,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v0.35.0-alpha.2@sha256:e751c20edbefb1e458d9272ec5faea011e41bb828c067a562d8ddbd8d9da7242" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v0.35.0-alpha.3@sha256:4a67ec40c13bf912151957dc0e7409f7e01a2859905285c98c20baae7b667ae2" certificates: commonName: "SeaweedFS CA" ipAddresses: [] From 432ddf6abcdb220791e2036356bad0806527d5bc Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Fri, 1 Aug 2025 18:52:56 +0200 Subject: [PATCH 20/30] [ci] Fix creating draft release Signed-off-by: Andrei Kvapil --- .github/workflows/tags.yaml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index e2418962..80100f23 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -149,36 +149,35 @@ jobs: version: ${{ steps.tag.outputs.tag }} # A compare-to: ${{ steps.latest_release.outputs.tag }} # B - # Create or reuse DRAFT GitHub Release + # Create or reuse draft release - name: Create / reuse draft release if: steps.check_release.outputs.skip == 'false' id: release uses: actions/github-script@v7 with: script: | - const tag = '${{ steps.tag.outputs.tag }}'; - const isRc = ${{ steps.tag.outputs.is_rc }}; - const outdated = '${{ steps.semver.outputs.comparison-result }}' === '<'; - const makeLatest = outdated ? false : 'legacy'; - const releases = await github.rest.repos.listReleases({ + const tag = '${{ steps.tag.outputs.tag }}'; + const isRc = ${{ steps.tag.outputs.is_rc }}; + const releases = await github.rest.repos.listReleases({ owner: context.repo.owner, repo: context.repo.repo }); - let rel = releases.data.find(r => r.tag_name === tag); + + let rel = releases.data.find(r => r.tag_name === tag); if (!rel) { rel = await github.rest.repos.createRelease({ owner: context.repo.owner, repo: context.repo.repo, - tag_name: tag, - name: tag, - draft: true, - prerelease: isRc, - make_latest: makeLatest + tag_name: tag, + name: tag, + draft: true, + prerelease: isRc // no make_latest for drafts }); console.log(`Draft release created for ${tag}`); } else { console.log(`Re-using existing release ${tag}`); } + core.setOutput('upload_url', rel.upload_url); # Build + upload assets (optional) From 44fabd4abcbebea765a7f86c87d9b9811b774f8c Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Mon, 4 Aug 2025 14:12:37 +0300 Subject: [PATCH 21/30] [virtual-machine] Disable instanceType validation Workaround for #1299. If a Cozystack installation provides custom instance types for virtual machines, the static validation rules prevent such instance types from being used, as they are included in the OpenAPI schema of the Cozystack API server and then once more applied in the dependent HelmRelease, offering users no easy way to remedy this in runtime. [virtual-machine] Disable instanceType validation to enable using custom instance types. Signed-off-by: Timofei Larkin --- packages/apps/versions_map | 6 ++- packages/apps/virtual-machine/Chart.yaml | 2 +- packages/apps/virtual-machine/Makefile | 4 +- .../apps/virtual-machine/values.schema.json | 53 +------------------ packages/apps/vm-instance/Chart.yaml | 2 +- packages/apps/vm-instance/Makefile | 4 +- packages/apps/vm-instance/values.schema.json | 53 +------------------ 7 files changed, 12 insertions(+), 112 deletions(-) diff --git a/packages/apps/versions_map b/packages/apps/versions_map index 95b80bb3..2b8cb331 100644 --- a/packages/apps/versions_map +++ b/packages/apps/versions_map @@ -174,7 +174,8 @@ virtual-machine 0.10.2 632224a3 virtual-machine 0.11.0 4369b031 virtual-machine 0.12.0 acd4663a virtual-machine 0.12.1 909208ba -virtual-machine 0.12.2 HEAD +virtual-machine 0.12.2 8ddbe32e +virtual-machine 0.12.3 HEAD vm-disk 0.1.0 d971f2ff vm-disk 0.1.1 6130f43d vm-disk 0.1.2 632224a3 @@ -193,7 +194,8 @@ vm-instance 0.7.2 632224a3 vm-instance 0.8.0 4369b031 vm-instance 0.9.0 acd4663a vm-instance 0.10.0 909208ba -vm-instance 0.10.1 HEAD +vm-instance 0.10.1 8ddbe32e +vm-instance 0.10.2 HEAD vpn 0.1.0 263e47be vpn 0.2.0 53f2365e vpn 0.3.0 6c5cf5bf diff --git a/packages/apps/virtual-machine/Chart.yaml b/packages/apps/virtual-machine/Chart.yaml index 68b5e986..e58f6a0c 100644 --- a/packages/apps/virtual-machine/Chart.yaml +++ b/packages/apps/virtual-machine/Chart.yaml @@ -17,7 +17,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.12.2 +version: 0.12.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/packages/apps/virtual-machine/Makefile b/packages/apps/virtual-machine/Makefile index 4bbfcd8c..5234bd85 100644 --- a/packages/apps/virtual-machine/Makefile +++ b/packages/apps/virtual-machine/Makefile @@ -3,8 +3,8 @@ include ../../../scripts/package.mk generate: readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md yq -o json -i '.properties.gpus.items.type = "object" | .properties.gpus.default = []' values.schema.json - INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \ - && yq -i -o json ".properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json + # INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \ + # && yq -i -o json ".properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json PREFERENCES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/preferences.yaml | yq 'split(" ") | . + [""]' -o json) \ && yq -i -o json ".properties.instanceProfile.enum = $${PREFERENCES}" values.schema.json yq -i -o json '.properties.externalPorts.items.type = "integer"' values.schema.json diff --git a/packages/apps/virtual-machine/values.schema.json b/packages/apps/virtual-machine/values.schema.json index 92b7da9e..2d328886 100644 --- a/packages/apps/virtual-machine/values.schema.json +++ b/packages/apps/virtual-machine/values.schema.json @@ -93,58 +93,7 @@ "instanceType": { "default": "u1.medium", "description": "Virtual Machine instance type", - "type": "string", - "enum": [ - "cx1.2xlarge", - "cx1.4xlarge", - "cx1.8xlarge", - "cx1.large", - "cx1.medium", - "cx1.xlarge", - "gn1.2xlarge", - "gn1.4xlarge", - "gn1.8xlarge", - "gn1.xlarge", - "m1.2xlarge", - "m1.4xlarge", - "m1.8xlarge", - "m1.large", - "m1.xlarge", - "n1.2xlarge", - "n1.4xlarge", - "n1.8xlarge", - "n1.large", - "n1.medium", - "n1.xlarge", - "o1.2xlarge", - "o1.4xlarge", - "o1.8xlarge", - "o1.large", - "o1.medium", - "o1.micro", - "o1.nano", - "o1.small", - "o1.xlarge", - "rt1.2xlarge", - "rt1.4xlarge", - "rt1.8xlarge", - "rt1.large", - "rt1.medium", - "rt1.micro", - "rt1.small", - "rt1.xlarge", - "u1.2xlarge", - "u1.2xmedium", - "u1.4xlarge", - "u1.8xlarge", - "u1.large", - "u1.medium", - "u1.micro", - "u1.nano", - "u1.small", - "u1.xlarge", - "" - ] + "type": "string" }, "resources": { "properties": { diff --git a/packages/apps/vm-instance/Chart.yaml b/packages/apps/vm-instance/Chart.yaml index 28733b49..3adb6a85 100644 --- a/packages/apps/vm-instance/Chart.yaml +++ b/packages/apps/vm-instance/Chart.yaml @@ -17,7 +17,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.10.1 +version: 0.10.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/packages/apps/vm-instance/Makefile b/packages/apps/vm-instance/Makefile index 3a21209b..38d781ae 100644 --- a/packages/apps/vm-instance/Makefile +++ b/packages/apps/vm-instance/Makefile @@ -4,8 +4,8 @@ generate: readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md yq -o json -i '.properties.disks.items.type = "object" | .properties.disks.default = []' values.schema.json yq -o json -i '.properties.gpus.items.type = "object" | .properties.gpus.default = []' values.schema.json - INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \ - && yq -i -o json ".properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json + #INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \ + # && yq -i -o json ".properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json PREFERENCES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/preferences.yaml | yq 'split(" ") | . + [""]' -o json) \ && yq -i -o json ".properties.instanceProfile.enum = $${PREFERENCES}" values.schema.json yq -i -o json '.properties.externalPorts.items.type = "integer"' values.schema.json diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json index 05b49a0d..c411e3ab 100644 --- a/packages/apps/vm-instance/values.schema.json +++ b/packages/apps/vm-instance/values.schema.json @@ -101,58 +101,7 @@ "instanceType": { "default": "u1.medium", "description": "Virtual Machine instance type", - "type": "string", - "enum": [ - "cx1.2xlarge", - "cx1.4xlarge", - "cx1.8xlarge", - "cx1.large", - "cx1.medium", - "cx1.xlarge", - "gn1.2xlarge", - "gn1.4xlarge", - "gn1.8xlarge", - "gn1.xlarge", - "m1.2xlarge", - "m1.4xlarge", - "m1.8xlarge", - "m1.large", - "m1.xlarge", - "n1.2xlarge", - "n1.4xlarge", - "n1.8xlarge", - "n1.large", - "n1.medium", - "n1.xlarge", - "o1.2xlarge", - "o1.4xlarge", - "o1.8xlarge", - "o1.large", - "o1.medium", - "o1.micro", - "o1.nano", - "o1.small", - "o1.xlarge", - "rt1.2xlarge", - "rt1.4xlarge", - "rt1.8xlarge", - "rt1.large", - "rt1.medium", - "rt1.micro", - "rt1.small", - "rt1.xlarge", - "u1.2xlarge", - "u1.2xmedium", - "u1.4xlarge", - "u1.8xlarge", - "u1.large", - "u1.medium", - "u1.micro", - "u1.nano", - "u1.small", - "u1.xlarge", - "" - ] + "type": "string" }, "resources": { "properties": { From 080289fa00fccc79476eae3d6b544eb939770b6d Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Mon, 4 Aug 2025 15:26:31 +0300 Subject: [PATCH 22/30] [kubernetes] Disable VPA for VPA in tenant k8s This patch disables the VPA for VPA deployment in tenant kubernetes clusters. This feature was never designed for deployment in tenant clusters and causes unexpected errors. [kubernetes] Disable VPA for VPA feature for tenant clusters, fixing an unintended regression. Signed-off-by: Timofei Larkin --- packages/apps/kubernetes/Chart.yaml | 2 +- .../templates/helmreleases/vertical-pod-autoscaler.yaml | 1 + packages/apps/versions_map | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/apps/kubernetes/Chart.yaml b/packages/apps/kubernetes/Chart.yaml index e0c4762e..79133ac1 100644 --- a/packages/apps/kubernetes/Chart.yaml +++ b/packages/apps/kubernetes/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.26.2 +version: 0.26.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml index 0d86b24e..9bb81ed8 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml @@ -4,6 +4,7 @@ {{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }} {{- $targetTenant := index $myNS.metadata.annotations "namespace.cozystack.io/monitoring" }} vertical-pod-autoscaler: + vpaForVPA: false recommender: extraArgs: container-name-label: container diff --git a/packages/apps/versions_map b/packages/apps/versions_map index 95b80bb3..022e4b1a 100644 --- a/packages/apps/versions_map +++ b/packages/apps/versions_map @@ -63,7 +63,8 @@ kubernetes 0.25.1 acd4663a kubernetes 0.25.2 08cb7c0f kubernetes 0.26.0 9584e5f5 kubernetes 0.26.1 0e47e1e8 -kubernetes 0.26.2 HEAD +kubernetes 0.26.2 8ddbe32e +kubernetes 0.26.3 HEAD mysql 0.1.0 263e47be mysql 0.2.0 c24a103f mysql 0.3.0 53f2365e From 85063cf624adc2130a0b8a1316b0ab2c0c655ff0 Mon Sep 17 00:00:00 2001 From: kklinch0 Date: Mon, 4 Aug 2025 18:21:11 +0300 Subject: [PATCH 23/30] clickhouse add chk Signed-off-by: kklinch0 --- hack/e2e-apps/clickhouse.bats | 4 + packages/apps/clickhouse/Chart.yaml | 2 +- packages/apps/clickhouse/Makefile | 1 + packages/apps/clickhouse/README.md | 8 + .../apps/clickhouse/templates/chkeeper.yaml | 96 + .../apps/clickhouse/templates/clickhouse.yaml | 6 + .../clickhouse/templates/workloadmonitor.yaml | 15 + packages/apps/clickhouse/values.schema.json | 31 + packages/apps/clickhouse/values.yaml | 8 + packages/apps/versions_map | 3 +- .../altinity-clickhouse-operator/Chart.yaml | 11 +- .../altinity-clickhouse-operator/README.md | 48 +- .../README.md.gotmpl | 17 + ...installations.clickhouse.altinity.com.yaml | 241 +- ...tiontemplates.clickhouse.altinity.com.yaml | 241 +- ...ations.clickhouse-keeper.altinity.com.yaml | 703 +- ...onfigurations.clickhouse.altinity.com.yaml | 91 +- ...ltinity_ClickHouse_Operator_dashboard.json | 7553 +++++++++-------- .../files/ClickHouseKeeper_dashboard.json | 1421 ++-- .../files/ClickHouse_Queries_dashboard.json | 6 +- .../templates/_helpers.tpl | 26 +- .../templates/dashboards-configmap.yaml | 21 + .../templates/dashboards-secret.yaml | 21 - ...rRole-clickhouse-operator-kube-system.yaml | 4 +- ...nding-clickhouse-operator-kube-system.yaml | 16 +- ...p-etc-clickhouse-operator-confd-files.yaml | 3 +- ...etc-clickhouse-operator-configd-files.yaml | 3 +- ...nfigMap-etc-clickhouse-operator-files.yaml | 3 +- ...-clickhouse-operator-templatesd-files.yaml | 3 +- ...-etc-clickhouse-operator-usersd-files.yaml | 3 +- ...igMap-etc-keeper-operator-confd-files.yaml | 14 + ...Map-etc-keeper-operator-configd-files.yaml | 14 + ...-etc-keeper-operator-templatesd-files.yaml | 14 + ...gMap-etc-keeper-operator-usersd-files.yaml | 14 + .../Deployment-clickhouse-operator.yaml | 60 +- .../generated/Role-clickhouse-operator.yaml | 211 + .../RoleBinding-clickhouse-operator.yaml | 23 + .../generated/Secret-clickhouse-operator.yaml | 5 +- .../Service-clickhouse-operator-metrics.yaml | 3 +- .../ServiceAccount-clickhouse-operator.yaml | 4 +- .../templates/servicemonitor.yaml | 35 +- .../altinity-clickhouse-operator/values.yaml | 253 +- 42 files changed, 6982 insertions(+), 4277 deletions(-) create mode 100644 packages/apps/clickhouse/templates/chkeeper.yaml create mode 100644 packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md.gotmpl create mode 100644 packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-configmap.yaml delete mode 100644 packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-secret.yaml create mode 100644 packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-confd-files.yaml create mode 100644 packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-configd-files.yaml create mode 100644 packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-templatesd-files.yaml create mode 100644 packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-usersd-files.yaml create mode 100644 packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Role-clickhouse-operator.yaml create mode 100644 packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/RoleBinding-clickhouse-operator.yaml diff --git a/hack/e2e-apps/clickhouse.bats b/hack/e2e-apps/clickhouse.bats index 3bfded10..1684c0c4 100644 --- a/hack/e2e-apps/clickhouse.bats +++ b/hack/e2e-apps/clickhouse.bats @@ -27,6 +27,10 @@ spec: s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog resticPassword: ChaXoveekoh6eigh4siesheeda2quai0 + clickhouseKeeper: + enabled: true + resourcesPreset: "micro" + size: "1Gi" resources: {} resourcesPreset: "nano" EOF diff --git a/packages/apps/clickhouse/Chart.yaml b/packages/apps/clickhouse/Chart.yaml index e9509471..66233d3f 100644 --- a/packages/apps/clickhouse/Chart.yaml +++ b/packages/apps/clickhouse/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.11.1 +version: 0.12.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/packages/apps/clickhouse/Makefile b/packages/apps/clickhouse/Makefile index aa6eaac6..b46aa5b8 100644 --- a/packages/apps/clickhouse/Makefile +++ b/packages/apps/clickhouse/Makefile @@ -7,6 +7,7 @@ include ../../../scripts/package.mk generate: readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md yq -i -o json --indent 4 '.properties.resourcesPreset.enum = $(PRESET_ENUM)' values.schema.json + yq -i -o json --indent 4 '.properties.clickhouseKeeper.resourcesPreset.enum = $(PRESET_ENUM)' values.schema.json image: docker buildx build images/clickhouse-backup \ diff --git a/packages/apps/clickhouse/README.md b/packages/apps/clickhouse/README.md index 25e896bf..370197b8 100644 --- a/packages/apps/clickhouse/README.md +++ b/packages/apps/clickhouse/README.md @@ -53,6 +53,14 @@ For more details, read [Restic: Effective Backup from Stdin](https://blog.aenix. | `backup.s3SecretKey` | Secret key for S3, used for authentication | `ju3eum4dekeich9ahM1te8waeGai0oog` | | `backup.resticPassword` | Password for Restic backup encryption | `ChaXoveekoh6eigh4siesheeda2quai0` | +### clickhouseKeeper parameters + +| Name | Description | Value | +| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | ------- | +| `clickhouseKeeper.enabled` | Deploy ClickHouse Keeper for cluster coordination | `true` | +| `clickhouseKeeper.size` | Persistent Volume Claim size, available for application data | `1Gi` | +| `clickhouseKeeper.resourcesPreset` | Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge. | `micro` | + ## Parameter examples and reference ### resources and resourcesPreset diff --git a/packages/apps/clickhouse/templates/chkeeper.yaml b/packages/apps/clickhouse/templates/chkeeper.yaml new file mode 100644 index 00000000..0739710f --- /dev/null +++ b/packages/apps/clickhouse/templates/chkeeper.yaml @@ -0,0 +1,96 @@ +{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} +{{- $clusterDomain := (index $cozyConfig.data "cluster-domain") | default "cozy.local" }} + +{{- if .Values.clickhouseKeeper.enabled }} +apiVersion: "clickhouse-keeper.altinity.com/v1" +kind: "ClickHouseKeeperInstallation" +metadata: + name: "{{ .Release.Name }}-keeper" + annotations: + prometheus.io/port: "7000" + prometheus.io/scrape: "true" +spec: + namespaceDomainPattern: "%s.svc.{{ $clusterDomain }}" + configuration: + clusters: + - name: "cluster1" + layout: + replicasCount: 3 + settings: + logger/level: "trace" + logger/console: "true" + listen_host: "0.0.0.0" + keeper_server/four_letter_word_white_list: "*" + keeper_server/coordination_settings/raft_logs_level: "information" + prometheus/endpoint: "/metrics" + prometheus/port: "7000" + prometheus/metrics: "true" + prometheus/events: "true" + prometheus/asynchronous_metrics: "true" + prometheus/status_info: "false" + + defaults: + templates: + # Templates are specified as default for all clusters + podTemplate: default + dataVolumeClaimTemplate: default + + templates: + podTemplates: + - name: default + metadata: + labels: + app: "{{ .Release.Name }}-keeper" + annotations: + prometheus.io/port: "7000" + prometheus.io/scrape: "true" + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - "{{ .Release.Name }}-keeper" + topologyKey: "kubernetes.io/hostname" + containers: + - name: clickhouse-keeper + imagePullPolicy: IfNotPresent + image: clickhouse/clickhouse-keeper:24.9.2.42 + resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.clickhouseKeeper.resourcesPreset .Values.resources $) | nindent 20 }} + securityContext: + fsGroup: 101 + + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "{{ .Values.clickhouseKeeper.size }}" +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: {{ .Release.Name }}-keeper + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: {{ .Release.Name }}-keeper + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + podMetricsEndpoints: + - port: metrics + path: /metrics + interval: 30s + scheme: http + relabelConfigs: + - action: replace + sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: instance +{{- end }} diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index 1acc7003..bab1fbe0 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -91,6 +91,12 @@ spec: layout: shardsCount: {{ .Values.shards }} replicasCount: {{ .Values.replicas }} + {{- if .Values.clickhouseKeeper.enabled }} + zookeeper: + nodes: + - host: "keeper-{{ .Release.Name }}-keeper.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}" + port: 2181 + {{- end }} templates: volumeClaimTemplates: - name: data-volume-template diff --git a/packages/apps/clickhouse/templates/workloadmonitor.yaml b/packages/apps/clickhouse/templates/workloadmonitor.yaml index f87db7bf..43c91672 100644 --- a/packages/apps/clickhouse/templates/workloadmonitor.yaml +++ b/packages/apps/clickhouse/templates/workloadmonitor.yaml @@ -11,3 +11,18 @@ spec: selector: app.kubernetes.io/instance: {{ $.Release.Name }} version: {{ $.Chart.Version }} +{{- if .Values.clickhouseKeeper.enabled }} +--- +apiVersion: cozystack.io/v1alpha1 +kind: WorkloadMonitor +metadata: + name: {{ $.Release.Name }}-keeper +spec: + replicas: 3 + minReplicas: 3 + kind: clickhouse + type: clickhouse + selector: + app: {{ $.Release.Name }}-keeper + version: {{ $.Chart.Version }} +{{- end }} diff --git a/packages/apps/clickhouse/values.schema.json b/packages/apps/clickhouse/values.schema.json index 397da6de..f76fc86f 100644 --- a/packages/apps/clickhouse/values.schema.json +++ b/packages/apps/clickhouse/values.schema.json @@ -45,6 +45,37 @@ }, "type": "object" }, + "clickhouseKeeper": { + "properties": { + "enabled": { + "default": true, + "description": "Deploy ClickHouse Keeper for cluster coordination ", + "type": "boolean" + }, + "resourcesPreset": { + "default": "micro", + "description": "Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge.", + "type": "string" + }, + "size": { + "default": "1Gi", + "description": "Persistent Volume Claim size, available for application data", + "type": "string" + } + }, + "type": "object", + "resourcesPreset": { + "enum": [ + "nano", + "micro", + "small", + "medium", + "large", + "xlarge", + "2xlarge" + ] + } + }, "logStorageSize": { "default": "2Gi", "description": "Size of Persistent Volume for logs", diff --git a/packages/apps/clickhouse/values.yaml b/packages/apps/clickhouse/values.yaml index dbe0ddae..76aaac95 100644 --- a/packages/apps/clickhouse/values.yaml +++ b/packages/apps/clickhouse/values.yaml @@ -56,3 +56,11 @@ backup: s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog resticPassword: ChaXoveekoh6eigh4siesheeda2quai0 +## @section clickhouseKeeper parameters +## @param clickhouseKeeper.enabled Deploy ClickHouse Keeper for cluster coordination +## @param clickhouseKeeper.size Persistent Volume Claim size, available for application data +## @param clickhouseKeeper.resourcesPreset Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge. +clickhouseKeeper: + enabled: true + size: 1Gi + resourcesPreset: micro diff --git a/packages/apps/versions_map b/packages/apps/versions_map index 95b80bb3..fc8c077e 100644 --- a/packages/apps/versions_map +++ b/packages/apps/versions_map @@ -15,7 +15,8 @@ clickhouse 0.9.2 632224a3 clickhouse 0.10.0 6358fd7a clickhouse 0.10.1 4369b031 clickhouse 0.11.0 08cb7c0f -clickhouse 0.11.1 HEAD +clickhouse 0.11.1 0e47e1e8 +clickhouse 0.12.0 HEAD ferretdb 0.1.0 e9716091 ferretdb 0.1.1 91b0499a ferretdb 0.2.0 6c5cf5bf diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/Chart.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/Chart.yaml index be7fe0c1..f68e1701 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/Chart.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/Chart.yaml @@ -1,11 +1,12 @@ apiVersion: v2 -appVersion: 0.23.4 +appVersion: 0.25.2 description: 'Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator). The ClickHouse Operator creates, configures and manages ClickHouse clusters running on Kubernetes. For upgrade please install CRDs separately: ```bash kubectl apply - -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml kubectl - apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml kubectl - apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml + -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml kubectl + apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml kubectl + apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml kubectl + apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml ```' home: https://github.com/Altinity/clickhouse-operator icon: https://logosandtypes.com/wp-content/uploads/2020/12/altinity.svg @@ -14,4 +15,4 @@ maintainers: name: altinity name: altinity-clickhouse-operator type: application -version: 0.23.4 +version: 0.25.2 diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md index d81cc7ca..e4a1ad68 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md @@ -1,6 +1,6 @@ # altinity-clickhouse-operator -![Version: 0.23.4](https://img.shields.io/badge/Version-0.23.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.23.4](https://img.shields.io/badge/AppVersion-0.23.4-informational?style=flat-square) +![Version: 0.25.2](https://img.shields.io/badge/Version-0.25.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.25.2](https://img.shields.io/badge/AppVersion-0.25.2-informational?style=flat-square) Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator). @@ -8,9 +8,10 @@ The ClickHouse Operator creates, configures and manages ClickHouse clusters runn For upgrade please install CRDs separately: ```bash - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml ``` **Homepage:** @@ -25,34 +26,38 @@ For upgrade please install CRDs separately: | Key | Type | Default | Description | |-----|------|---------|-------------| -| additionalResources | list | `[]` | list of additional resources to create (are processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator, look `kubectl explain chi` for details | -| affinity | object | `{}` | affinity for scheduler pod assignment, look `kubectl explain pod.spec.affinity` for details | -| configs | object | check the values.yaml file for the config content, auto-generated from latest operator release | clickhouse-operator configs | +| additionalResources | list | `[]` | list of additional resources to create (processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator. check `kubectl explain chi` for details | +| affinity | object | `{}` | affinity for scheduler pod assignment, check `kubectl explain pod.spec.affinity` for details | +| commonAnnotations | object | `{}` | set of annotations that will be applied to all the resources for the operator | +| commonLabels | object | `{}` | set of labels that will be applied to all the resources for the operator | +| configs | object | check the `values.yaml` file for the config content (auto-generated from latest operator release) | clickhouse operator configs | | dashboards.additionalLabels | object | `{"grafana_dashboard":""}` | labels to add to a secret with dashboards | | dashboards.annotations | object | `{}` | annotations to add to a secret with dashboards | -| dashboards.enabled | bool | `false` | provision grafana dashboards as secrets (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-6.33.1/charts/grafana/values.yaml#L679 ) | +| dashboards.enabled | bool | `false` | provision grafana dashboards as configMaps (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-8.3.4/charts/grafana/values.yaml#L778 ) | | dashboards.grafana_folder | string | `"clickhouse"` | | | fullnameOverride | string | `""` | full name of the chart. | -| imagePullSecrets | list | `[]` | image pull secret for private images in clickhouse-operator pod possible value format [{"name":"your-secret-name"}] look `kubectl explain pod.spec.imagePullSecrets` for details | +| imagePullSecrets | list | `[]` | image pull secret for private images in clickhouse-operator pod possible value format `[{"name":"your-secret-name"}]`, check `kubectl explain pod.spec.imagePullSecrets` for details | | metrics.containerSecurityContext | object | `{}` | | | metrics.enabled | bool | `true` | | -| metrics.env | list | `[]` | additional environment variables for the deployment of metrics-exporter containers possible format value [{"name": "SAMPLE", "value": "text"}] | +| metrics.env | list | `[]` | additional environment variables for the deployment of metrics-exporter containers possible format value `[{"name": "SAMPLE", "value": "text"}]` | | metrics.image.pullPolicy | string | `"IfNotPresent"` | image pull policy | | metrics.image.repository | string | `"altinity/metrics-exporter"` | image repository | | metrics.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) | | metrics.resources | object | `{}` | custom resource configuration | | nameOverride | string | `""` | override name of the chart | -| nodeSelector | object | `{}` | node for scheduler pod assignment, look `kubectl explain pod.spec.nodeSelector` for details | +| namespaceOverride | string | `""` | | +| nodeSelector | object | `{}` | node for scheduler pod assignment, check `kubectl explain pod.spec.nodeSelector` for details | | operator.containerSecurityContext | object | `{}` | | -| operator.env | list | `[]` | additional environment variables for the clickhouse-operator container in deployment possible format value [{"name": "SAMPLE", "value": "text"}] | +| operator.env | list | `[]` | additional environment variables for the clickhouse-operator container in deployment possible format value `[{"name": "SAMPLE", "value": "text"}]` | | operator.image.pullPolicy | string | `"IfNotPresent"` | image pull policy | | operator.image.repository | string | `"altinity/clickhouse-operator"` | image repository | | operator.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) | -| operator.resources | object | `{}` | custom resource configuration, look `kubectl explain pod.spec.containers.resources` for details | -| podAnnotations | object | `{"clickhouse-operator-metrics/port":"9999","clickhouse-operator-metrics/scrape":"true","prometheus.io/port":"8888","prometheus.io/scrape":"true"}` | annotations to add to the clickhouse-operator pod, look `kubectl explain pod.spec.annotations` for details | +| operator.resources | object | `{}` | custom resource configuration, check `kubectl explain pod.spec.containers.resources` for details | +| podAnnotations | object | check the `values.yaml` file | annotations to add to the clickhouse-operator pod, check `kubectl explain pod.spec.annotations` for details | | podLabels | object | `{}` | labels to add to the clickhouse-operator pod | | podSecurityContext | object | `{}` | | -| rbac.create | bool | `true` | specifies whether cluster roles and cluster role bindings should be created | +| rbac.create | bool | `true` | specifies whether rbac resources should be created | +| rbac.namespaceScoped | bool | `false` | specifies whether to create roles and rolebindings at the cluster level or namespace level | | secret.create | bool | `true` | create a secret with operator credentials | | secret.password | string | `"clickhouse_operator_password"` | operator credentials password | | secret.username | string | `"clickhouse_operator"` | operator credentials username | @@ -60,6 +65,15 @@ For upgrade please install CRDs separately: | serviceAccount.create | bool | `true` | specifies whether a service account should be created | | serviceAccount.name | string | `nil` | the name of the service account to use; if not set and create is true, a name is generated using the fullname template | | serviceMonitor.additionalLabels | object | `{}` | additional labels for service monitor | -| serviceMonitor.enabled | bool | `false` | ServiceMonitor Custom resource is created for a (prometheus-operator)[https://github.com/prometheus-operator/prometheus-operator] | -| tolerations | list | `[]` | tolerations for scheduler pod assignment, look `kubectl explain pod.spec.tolerations` for details | +| serviceMonitor.clickhouseMetrics.interval | string | `"30s"` | | +| serviceMonitor.clickhouseMetrics.metricRelabelings | list | `[]` | | +| serviceMonitor.clickhouseMetrics.relabelings | list | `[]` | | +| serviceMonitor.clickhouseMetrics.scrapeTimeout | string | `""` | | +| serviceMonitor.enabled | bool | `false` | ServiceMonitor Custom resource is created for a [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) In serviceMonitor will be created two endpoints clickhouse-metrics on port 8888 and operator-metrics # 9999. Ypu can specify interval, scrapeTimeout, relabelings, metricRelabelings for each endpoint below | +| serviceMonitor.operatorMetrics.interval | string | `"30s"` | | +| serviceMonitor.operatorMetrics.metricRelabelings | list | `[]` | | +| serviceMonitor.operatorMetrics.relabelings | list | `[]` | | +| serviceMonitor.operatorMetrics.scrapeTimeout | string | `""` | | +| tolerations | list | `[]` | tolerations for scheduler pod assignment, check `kubectl explain pod.spec.tolerations` for details | +| topologySpreadConstraints | list | `[]` | | diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md.gotmpl b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md.gotmpl new file mode 100644 index 00000000..8dff3784 --- /dev/null +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/README.md.gotmpl @@ -0,0 +1,17 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml index f3eff275..84359a4b 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.25.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.25.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -51,13 +51,12 @@ spec: jsonPath: .status.taskID - name: status type: string - description: CHI status + description: Resource status jsonPath: .status.status - - name: hosts-unchanged + - name: hosts-completed type: integer - description: Unchanged hosts count - priority: 1 # show in wide view - jsonPath: .status.hostsUnchanged + description: Completed hosts count + jsonPath: .status.hostsCompleted - name: hosts-updated type: integer description: Updated hosts count @@ -68,20 +67,11 @@ spec: description: Added hosts count priority: 1 # show in wide view jsonPath: .status.hostsAdded - - name: hosts-completed - type: integer - description: Completed hosts count - jsonPath: .status.hostsCompleted - name: hosts-deleted type: integer description: Hosts deleted count priority: 1 # show in wide view jsonPath: .status.hostsDeleted - - name: hosts-delete - type: integer - description: Hosts to be deleted count - priority: 1 # show in wide view - jsonPath: .status.hostsDelete - name: endpoint type: string description: Client access endpoint @@ -92,39 +82,51 @@ spec: description: Age of the resource # Displayed in all priorities jsonPath: .metadata.creationTimestamp + - name: suspend + type: string + description: Suspend reconciliation + # Displayed in all priorities + jsonPath: .spec.suspend subresources: status: {} schema: openAPIV3Schema: - description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" type: object required: - spec properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object status: type: object - description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + description: | + Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other properties: chop-version: type: string - description: "ClickHouse operator version" + description: "Operator version" chop-commit: type: string - description: "ClickHouse operator git commit SHA" + description: "Operator git commit SHA" chop-date: type: string - description: "ClickHouse operator build date" + description: "Operator build date" chop-ip: type: string - description: "IP address of the operator's pod which managed this CHI" + description: "IP address of the operator's pod which managed this resource" clusters: type: integer minimum: 0 @@ -222,17 +224,23 @@ spec: endpoint: type: string description: "Endpoint" + endpoints: + type: array + description: "All endpoints" + nullable: true + items: + type: string generation: type: integer minimum: 0 description: "Generation" normalized: type: object - description: "Normalized CHI requested" + description: "Normalized resource requested" x-kubernetes-preserve-unknown-fields: true normalizedCompleted: type: object - description: "Normalized CHI completed" + description: "Normalized resource completed" x-kubernetes-preserve-unknown-fields: true hostsWithTablesCreated: type: array @@ -240,6 +248,12 @@ spec: nullable: true items: type: string + hostsWithReplicaCaughtUp: + type: array + description: "List of hosts with replica caught up" + nullable: true + items: + type: string usedTemplates: type: array description: "List of templates used to build this CHI" @@ -301,6 +315,13 @@ spec: enum: - "" - "RollingUpdate" + suspend: + !!merge <<: *TypeStringBool + description: | + Suspend reconciliation of resources managed by a ClickHouse Installation. + Works as the following: + - When `suspend` is `true` operator stops reconciling all resources. + - When `suspend` is `false` or not set, operator reconciles all resources. troubleshoot: !!merge <<: *TypeStringBool description: | @@ -412,6 +433,63 @@ spec: service: !!merge <<: *TypeObjectsCleanup description: "Behavior policy for failed Service, `Retain` by default" + runtime: + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." + macros: + type: object + description: "macros parameters" + properties: + sections: + type: object + description: "sections behaviour for macros" + properties: + users: + type: object + description: "sections behaviour for macros on users" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + profiles: + type: object + description: "sections behaviour for macros on profiles" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + quotas: + type: object + description: "sections behaviour for macros on quotas" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + settings: + type: object + description: "sections behaviour for macros on settings" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + files: + type: object + description: "sections behaviour for macros on files" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" defaults: type: object description: | @@ -424,7 +502,7 @@ spec: description: | define should replicas be specified by FQDN in ``. In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup - "yes" by default + "no" by default distributedDDL: type: object description: | @@ -474,7 +552,13 @@ spec: description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" serviceTemplate: type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + description: "optional, template name from chi.spec.templates.serviceTemplates. used for customization of the `Service` resource, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + serviceTemplates: + type: array + description: "optional, template names from chi.spec.templates.serviceTemplates. used for customization of the `Service` resources, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + nullable: true + items: + type: string clusterServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" @@ -486,7 +570,7 @@ spec: description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" volumeClaimTemplate: type: string - description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" configuration: type: object description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" @@ -521,6 +605,9 @@ spec: secure: !!merge <<: *TypeStringBool description: "if a secure connection to Zookeeper is required" + availabilityZone: + type: string + description: "availability zone for Zookeeper node" session_timeout_ms: type: integer description: "session timeout during connect to Zookeeper" @@ -540,6 +627,20 @@ spec: you can configure password hashed, authorization restrictions, database level security row filters etc. More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write directly into XML tag during render *-usersd ConfigMap + + any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write into environment variable and write to XML tag via from_env=XXX + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples # nullable: true x-kubernetes-preserve-unknown-fields: true profiles: @@ -566,6 +667,12 @@ spec: allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + secret value will pass in `pod.spec.env`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle # nullable: true x-kubernetes-preserve-unknown-fields: true files: &TypeFiles @@ -575,14 +682,20 @@ spec: every key in this object is the file name every value in this object is the file content you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html - each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + each key could contains prefix like {common}, {users}, {hosts} or config.d, users.d, conf.d, wrong prefixes will be ignored, subfolders also will be ignored More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + + any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets + secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/ + and will automatically update when update secret + it useful for pass SSL certificates from cert-manager or similar tool + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples # nullable: true x-kubernetes-preserve-unknown-fields: true clusters: type: array description: | - describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + describes clusters layout and allows change settings on cluster-level, shard-level and replica-level every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ @@ -595,7 +708,7 @@ spec: properties: name: type: string - description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" minLength: 1 # See namePartClusterMaxLen const maxLength: 15 @@ -683,6 +796,32 @@ spec: required: - name - key + pdbMaxUnavailable: + type: integer + description: | + Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions + by specifying 0. This is a mutually exclusive setting with "minAvailable". + minimum: 0 + maximum: 65535 + reconcile: + type: object + description: "allow tuning reconciling process" + properties: + runtime: + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." layout: type: object description: | @@ -690,18 +829,24 @@ spec: allows override settings on each shard and replica separatelly # nullable: true properties: - type: - type: string - description: "DEPRECATED - to be removed soon" shardsCount: type: integer - description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + description: | + how much shards for current ClickHouse cluster will run in Kubernetes, + each shard contains shared-nothing part of data and contains set of replicas, + cluster contains 1 shard by default" replicasCount: type: integer - description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + description: | + how much replicas in each shards for current cluster will run in Kubernetes, + each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + every shard contains 1 replica by default" shards: type: array - description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + description: | + optional, allows override top-level `chi.spec.configuration`, cluster-level + `chi.spec.configuration.clusters` settings for each shard separately, + use it only if you fully understand what you do" # nullable: true items: type: object @@ -1036,7 +1181,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1108,7 +1253,9 @@ spec: maximum: 65535 topologyKey: type: string - description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + description: | + use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, + more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" metadata: type: object description: | @@ -1124,7 +1271,8 @@ spec: x-kubernetes-preserve-unknown-fields: true volumeClaimTemplates: type: array - description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + description: | + allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else # nullable: true items: type: object @@ -1177,14 +1325,17 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: | + allows define format for generated `Service` name, + look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + for details about available template variables" metadata: # TODO specify ObjectMeta type: object description: | allows pass standard object's metadata from template to Service Could be use for define specificly for Cloud Provider metadata which impact to behavior of service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: @@ -1197,7 +1348,9 @@ spec: x-kubernetes-preserve-unknown-fields: true useTemplates: type: array - description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + description: | + list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI` + manifest during render Kubernetes resources to create related ClickHouse clusters" # nullable: true items: type: object diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml index d8ef8ba5..0d7dab1a 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.25.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.25.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -51,13 +51,12 @@ spec: jsonPath: .status.taskID - name: status type: string - description: CHI status + description: Resource status jsonPath: .status.status - - name: hosts-unchanged + - name: hosts-completed type: integer - description: Unchanged hosts count - priority: 1 # show in wide view - jsonPath: .status.hostsUnchanged + description: Completed hosts count + jsonPath: .status.hostsCompleted - name: hosts-updated type: integer description: Updated hosts count @@ -68,20 +67,11 @@ spec: description: Added hosts count priority: 1 # show in wide view jsonPath: .status.hostsAdded - - name: hosts-completed - type: integer - description: Completed hosts count - jsonPath: .status.hostsCompleted - name: hosts-deleted type: integer description: Hosts deleted count priority: 1 # show in wide view jsonPath: .status.hostsDeleted - - name: hosts-delete - type: integer - description: Hosts to be deleted count - priority: 1 # show in wide view - jsonPath: .status.hostsDelete - name: endpoint type: string description: Client access endpoint @@ -92,39 +82,51 @@ spec: description: Age of the resource # Displayed in all priorities jsonPath: .metadata.creationTimestamp + - name: suspend + type: string + description: Suspend reconciliation + # Displayed in all priorities + jsonPath: .spec.suspend subresources: status: {} schema: openAPIV3Schema: - description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" type: object required: - spec properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object status: type: object - description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + description: | + Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other properties: chop-version: type: string - description: "ClickHouse operator version" + description: "Operator version" chop-commit: type: string - description: "ClickHouse operator git commit SHA" + description: "Operator git commit SHA" chop-date: type: string - description: "ClickHouse operator build date" + description: "Operator build date" chop-ip: type: string - description: "IP address of the operator's pod which managed this CHI" + description: "IP address of the operator's pod which managed this resource" clusters: type: integer minimum: 0 @@ -222,17 +224,23 @@ spec: endpoint: type: string description: "Endpoint" + endpoints: + type: array + description: "All endpoints" + nullable: true + items: + type: string generation: type: integer minimum: 0 description: "Generation" normalized: type: object - description: "Normalized CHI requested" + description: "Normalized resource requested" x-kubernetes-preserve-unknown-fields: true normalizedCompleted: type: object - description: "Normalized CHI completed" + description: "Normalized resource completed" x-kubernetes-preserve-unknown-fields: true hostsWithTablesCreated: type: array @@ -240,6 +248,12 @@ spec: nullable: true items: type: string + hostsWithReplicaCaughtUp: + type: array + description: "List of hosts with replica caught up" + nullable: true + items: + type: string usedTemplates: type: array description: "List of templates used to build this CHI" @@ -301,6 +315,13 @@ spec: enum: - "" - "RollingUpdate" + suspend: + !!merge <<: *TypeStringBool + description: | + Suspend reconciliation of resources managed by a ClickHouse Installation. + Works as the following: + - When `suspend` is `true` operator stops reconciling all resources. + - When `suspend` is `false` or not set, operator reconciles all resources. troubleshoot: !!merge <<: *TypeStringBool description: | @@ -412,6 +433,63 @@ spec: service: !!merge <<: *TypeObjectsCleanup description: "Behavior policy for failed Service, `Retain` by default" + runtime: + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." + macros: + type: object + description: "macros parameters" + properties: + sections: + type: object + description: "sections behaviour for macros" + properties: + users: + type: object + description: "sections behaviour for macros on users" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + profiles: + type: object + description: "sections behaviour for macros on profiles" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + quotas: + type: object + description: "sections behaviour for macros on quotas" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + settings: + type: object + description: "sections behaviour for macros on settings" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + files: + type: object + description: "sections behaviour for macros on files" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" defaults: type: object description: | @@ -424,7 +502,7 @@ spec: description: | define should replicas be specified by FQDN in ``. In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup - "yes" by default + "no" by default distributedDDL: type: object description: | @@ -474,7 +552,13 @@ spec: description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" serviceTemplate: type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + description: "optional, template name from chi.spec.templates.serviceTemplates. used for customization of the `Service` resource, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + serviceTemplates: + type: array + description: "optional, template names from chi.spec.templates.serviceTemplates. used for customization of the `Service` resources, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + nullable: true + items: + type: string clusterServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" @@ -486,7 +570,7 @@ spec: description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" volumeClaimTemplate: type: string - description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" configuration: type: object description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" @@ -521,6 +605,9 @@ spec: secure: !!merge <<: *TypeStringBool description: "if a secure connection to Zookeeper is required" + availabilityZone: + type: string + description: "availability zone for Zookeeper node" session_timeout_ms: type: integer description: "session timeout during connect to Zookeeper" @@ -540,6 +627,20 @@ spec: you can configure password hashed, authorization restrictions, database level security row filters etc. More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write directly into XML tag during render *-usersd ConfigMap + + any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write into environment variable and write to XML tag via from_env=XXX + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples # nullable: true x-kubernetes-preserve-unknown-fields: true profiles: @@ -566,6 +667,12 @@ spec: allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + secret value will pass in `pod.spec.env`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle # nullable: true x-kubernetes-preserve-unknown-fields: true files: &TypeFiles @@ -575,14 +682,20 @@ spec: every key in this object is the file name every value in this object is the file content you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html - each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + each key could contains prefix like {common}, {users}, {hosts} or config.d, users.d, conf.d, wrong prefixes will be ignored, subfolders also will be ignored More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + + any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets + secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/ + and will automatically update when update secret + it useful for pass SSL certificates from cert-manager or similar tool + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples # nullable: true x-kubernetes-preserve-unknown-fields: true clusters: type: array description: | - describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + describes clusters layout and allows change settings on cluster-level, shard-level and replica-level every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ @@ -595,7 +708,7 @@ spec: properties: name: type: string - description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" minLength: 1 # See namePartClusterMaxLen const maxLength: 15 @@ -683,6 +796,32 @@ spec: required: - name - key + pdbMaxUnavailable: + type: integer + description: | + Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions + by specifying 0. This is a mutually exclusive setting with "minAvailable". + minimum: 0 + maximum: 65535 + reconcile: + type: object + description: "allow tuning reconciling process" + properties: + runtime: + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." layout: type: object description: | @@ -690,18 +829,24 @@ spec: allows override settings on each shard and replica separatelly # nullable: true properties: - type: - type: string - description: "DEPRECATED - to be removed soon" shardsCount: type: integer - description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + description: | + how much shards for current ClickHouse cluster will run in Kubernetes, + each shard contains shared-nothing part of data and contains set of replicas, + cluster contains 1 shard by default" replicasCount: type: integer - description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + description: | + how much replicas in each shards for current cluster will run in Kubernetes, + each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + every shard contains 1 replica by default" shards: type: array - description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + description: | + optional, allows override top-level `chi.spec.configuration`, cluster-level + `chi.spec.configuration.clusters` settings for each shard separately, + use it only if you fully understand what you do" # nullable: true items: type: object @@ -1036,7 +1181,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1108,7 +1253,9 @@ spec: maximum: 65535 topologyKey: type: string - description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + description: | + use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, + more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" metadata: type: object description: | @@ -1124,7 +1271,8 @@ spec: x-kubernetes-preserve-unknown-fields: true volumeClaimTemplates: type: array - description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + description: | + allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else # nullable: true items: type: object @@ -1177,14 +1325,17 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: | + allows define format for generated `Service` name, + look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + for details about available template variables" metadata: # TODO specify ObjectMeta type: object description: | allows pass standard object's metadata from template to Service Could be use for define specificly for Cloud Provider metadata which impact to behavior of service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: @@ -1197,7 +1348,9 @@ spec: x-kubernetes-preserve-unknown-fields: true useTemplates: type: array - description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + description: | + list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI` + manifest during render Kubernetes resources to create related ClickHouse clusters" # nullable: true items: type: object diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml index 07fdeca3..4581ff39 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml @@ -1,13 +1,13 @@ # Template Parameters: # -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.25.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.4 + clickhouse-keeper.altinity.com/chop: 0.25.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -22,123 +22,487 @@ spec: served: true storage: true additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID - name: status type: string - description: CHK status + description: Resource status jsonPath: .status.status - - name: replicas + - name: hosts-unchanged type: integer - description: Replica count + description: Unchanged hosts count priority: 1 # show in wide view - jsonPath: .status.replicas + jsonPath: .status.hostsUnchanged + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint - name: age type: date description: Age of the resource # Displayed in all priorities jsonPath: .metadata.creationTimestamp + - name: suspend + type: string + description: Suspend reconciliation + # Displayed in all priorities + jsonPath: .spec.suspend subresources: status: {} schema: openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" type: object required: - spec - description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one ClickHouse Keeper cluster" properties: apiVersion: - type: string description: | APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - kind: type: string + kind: description: | Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string metadata: type: object status: type: object description: | - Current ClickHouseKeeperInstallation status, contains many fields like overall status, desired replicas and ready replica list with their endpoints + Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other properties: chop-version: type: string - description: "ClickHouse operator version" + description: "Operator version" chop-commit: type: string - description: "ClickHouse operator git commit SHA" + description: "Operator git commit SHA" chop-date: type: string - description: "ClickHouse operator build date" + description: "Operator build date" chop-ip: type: string - description: "IP address of the operator's pod which managed this CHI" + description: "IP address of the operator's pod which managed this resource" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" status: type: string description: "Status" - replicas: - type: integer - format: int32 - description: Replicas is the number of number of desired replicas in the cluster - readyReplicas: + taskID: + type: string + description: "Current task id" + taskIDsStarted: type: array - description: ReadyReplicas is the array of endpoints of those ready replicas in the cluster + description: "Started task ids" + nullable: true items: - type: object - properties: - host: - type: string - description: dns name or ip address for Keeper node - port: - type: integer - minimum: 0 - maximum: 65535 - description: TCP port which used to connect to Keeper node - secure: - type: string - description: if a secure connection to Keeper is required + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUnchanged: + type: integer + minimum: 0 + description: "Unchanged Hosts count" + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + endpoints: + type: array + description: "All endpoints" + nullable: true + items: + type: string + generation: + type: integer + minimum: 0 + description: "Generation" normalized: type: object - description: "Normalized CHK requested" + description: "Normalized resource requested" x-kubernetes-preserve-unknown-fields: true normalizedCompleted: type: object - description: "Normalized CHK completed" + description: "Normalized resource completed" x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + hostsWithReplicaCaughtUp: + type: array + description: "List of hosts with replica caught up" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true spec: type: object - description: KeeperSpec defines the desired state of a Keeper cluster + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + suspend: + !!merge <<: *TypeStringBool + description: | + Suspend reconciliation of resources managed by a ClickHouse Keeper. + Works as the following: + - When `suspend` is `true` operator stops reconciling all resources. + - When `suspend` is `false` or not set, operator reconciles all resources. namespaceDomainPattern: type: string description: | Custom domain pattern which will be used for DNS names of `Service` or `Pod`. Typical use scenario - custom cluster domain in Kubernetes cluster Example: %s.svc.my.test - replicas: - type: integer - format: int32 + reconciling: + type: object + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + enum: + - "" + - "wait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + defaults: + type: object description: | - Replicas is the expected size of the keeper cluster. - The valid range of size is from 1 to 7. - minimum: 1 - maximum: 7 + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "no" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates. used for customization of the `Service` resource, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + serviceTemplates: + type: array + description: "optional, template names from chi.spec.templates.serviceTemplates. used for customization of the `Service` resources, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + nullable: true + items: + type: string + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" configuration: type: object description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" # nullable: true properties: - settings: + settings: &TypeSettings type: object - description: "allows configure multiple aspects and behavior for `clickhouse-keeper` instance" + description: | + allows configure multiple aspects and behavior for `clickhouse-keeper` instance + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting + # nullable: true x-kubernetes-preserve-unknown-fields: true clusters: type: array description: | - describes ClickHouseKeeper clusters layout and allows change settings on cluster-level and replica-level + describes clusters layout and allows change settings on cluster-level and replica-level # nullable: true items: type: object @@ -147,25 +511,178 @@ spec: properties: name: type: string - description: "cluster name, used to identify set of ClickHouseKeeper servers and wide used during generate names of related Kubernetes resources" + description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" minLength: 1 # See namePartClusterMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` layout: type: object description: | - describe current cluster layout, how many replicas + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly # nullable: true properties: replicasCount: type: integer - description: "how many replicas in ClickHouseKeeper cluster" + description: | + how much replicas in each shards for current cluster will run in Kubernetes, + each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + every shard contains 1 replica by default" + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zkPort: + type: integer + minimum: 1 + maximum: 65535 + raftPort: + type: integer + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` templates: type: object description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" # nullable: true properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zkPort: + type: integer + minimum: 1 + maximum: 65535 + raftPort: + type: integer + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" podTemplates: type: array description: | @@ -180,6 +697,83 @@ spec: name: type: string description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: | + use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, + more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" metadata: type: object description: | @@ -195,7 +789,8 @@ spec: x-kubernetes-preserve-unknown-fields: true volumeClaimTemplates: type: array - description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + description: | + allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else # nullable: true items: type: object @@ -211,6 +806,8 @@ spec: cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy metadata: type: object description: | @@ -244,6 +841,12 @@ spec: cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: | + allows define format for generated `Service` name, + look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index b53ef91d..9bfe368d 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -7,7 +7,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.25.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -137,6 +137,7 @@ spec: items: type: object description: "setting: value pairs for configuration restart policy" + x-kubernetes-preserve-unknown-fields: true access: type: object description: "parameters which use for connect to clickhouse from clickhouse-operator deployment" @@ -181,6 +182,47 @@ spec: minimum: 1 maximum: 600 description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds." + addons: + type: object + description: "Configuration addons specifies additional settings" + properties: + rules: + type: array + description: "Array of set of rules per specified ClickHouse versions" + items: + type: object + properties: + version: + type: string + description: "ClickHouse version expression" + spec: + type: object + description: "spec" + properties: + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + properties: + users: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true + settings: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true metrics: type: object description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator" @@ -323,6 +365,19 @@ spec: include: !!merge <<: *TypeStringBool description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" + replicas: + type: object + description: "Whether the operator during reconcile procedure should wait for replicas to catch-up" + properties: + all: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for all replicas to catch-up" + new: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for new replicas to catch-up" + delay: + type: integer + description: "replication max absolute delay to consider replica is not delayed" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -373,6 +428,40 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/Altinity_ClickHouse_Operator_dashboard.json b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/Altinity_ClickHouse_Operator_dashboard.json index 3d914458..0f3e34b8 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/Altinity_ClickHouse_Operator_dashboard.json +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/Altinity_ClickHouse_Operator_dashboard.json @@ -1,56 +1,23 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "clickhouse-operator-prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "7.5.15" - }, - { - "type": "panel", - "id": "grafana-piechart-panel", - "name": "Pie Chart", - "version": "1.6.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - }, - { - "type": "panel", - "id": "table-old", - "name": "Table (old)", - "version": "" - } - ], "annotations": { "list": [ { - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "enable": true, "expr": "ALERTS{app=~\"clickhouse-operator|zookeeper\"}", "hide": false, @@ -64,40 +31,87 @@ "textFormat": "{{alertstate}}", "titleFormat": "{{alertname}}", "type": "tags" - }, - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" } ] }, "description": "Alitinity Clickhouse Operator metrics exported by Monitoring Agent", "editable": true, + "fiscalYearStartMonth": 0, "gnetId": 882, "graphTooltip": 1, - "id": null, - "iteration": 1662652674457, + "id": 82, "links": [], + "liveNow": false, "panels": [ { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "fieldConfig": { - "defaults": {}, - "overrides": [] + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "hidden", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 43, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "auto" + } + ] + } + ] }, - "filterNull": false, - "fontSize": "100%", "gridPos": { "h": 4, "w": 10, @@ -105,55 +119,40 @@ "y": 0 }, "id": 15, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 2, - "desc": false - }, - "styles": [ - { - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - { - "align": "auto", - "colorMode": "value", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 1, - "pattern": "/.*/", - "thresholds": [ - "3600", - "86400" - ], - "type": "number", - "unit": "s" + "tooltip": { + "mode": "multi", + "sort": "none" } - ], + }, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_Uptime{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sort(avg by (hostname)(chi_clickhouse_metric_Uptime{chi=~\"$chi\",hostname=~\"$hostname\"})) OR on () vector(0)", + "interval": "", "intervalFactor": 2, "legendFormat": "{{hostname}}", "metric": "chi_clickhouse_metric_Uptime", + "range": true, "refId": "A", "step": 60 } ], - "title": "Uptime", - "transform": "timeseries_aggregations", - "type": "table-old" + "title": "Uptime (logarithmic)", + "transformations": [], + "type": "timeseries" }, { - "cacheTimeout": null, "colorBackground": false, "colorValue": true, "colors": [ @@ -161,12 +160,29 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Clickhouse operator metrics-exporter fails when grab metrics from clickhouse-server\n\nPlease look pods status\n\nkubectl get pods --all-namespaces | grep clickhouse", "editable": true, "error": false, "fieldConfig": { - "defaults": {}, + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, "overrides": [] }, "format": "none", @@ -185,7 +201,6 @@ }, "id": 47, "interval": "", - "isNew": true, "links": [ { "targetBlank": true, @@ -206,7 +221,23 @@ ], "maxDataPoints": 100, "nullPointMode": "connected", - "nullText": null, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.3", "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -228,7 +259,10 @@ "tableColumn": "", "targets": [ { - "expr": "sum(chi_clickhouse_metric_fetch_errors{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\",fetch_type=\"system.metrics\"})", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(chi_clickhouse_metric_fetch_errors{chi=~\"$chi\",hostname=~\"$hostname\",fetch_type=\"system.metrics\"})", "intervalFactor": 2, "legendFormat": "", "refId": "A", @@ -236,10 +270,8 @@ } ], "thresholds": "1,1", - "timeFrom": null, - "timeShift": null, "title": "Failed Pods", - "type": "singlestat", + "type": "stat", "valueFontSize": "80%", "valueMaps": [ { @@ -251,70 +283,311 @@ "valueName": "current" }, { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "For example, version 11.22.33 is translated to 11022033", "fieldConfig": { - "defaults": {}, - "overrides": [] + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "hidden", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 3, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "auto" + } + ] + } + ] }, - "filterNull": false, - "fontSize": "100%", "gridPos": { "h": 4, - "w": 11, + "w": 7, "x": 13, "y": 0 }, "hideTimeOverride": false, "id": 17, - "links": [], - "pageSize": null, - "scroll": false, - "showHeader": true, - "sort": { - "col": 3, - "desc": true - }, - "styles": [ - { - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 0, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "none" + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - ], + }, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_VersionInteger{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sort_desc(max by (hostname) (chi_clickhouse_metric_VersionInteger{chi=~\"$chi\",hostname=~\"$hostname\"}))", "intervalFactor": 2, "legendFormat": "{{hostname}}", "metric": "chi_clickhouse_metric_VersionInteger", + "range": true, "refId": "A", "step": 60 } ], - "timeFrom": null, - "timeShift": null, "title": "Version", - "transform": "timeseries_aggregations", - "type": "table-old" + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 0 + }, + "id": 56, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(chi_clickhouse_metric_NumberOfTables{chi=~\"$chi\",hostname=~\"$hostname\"})", + "instant": false, + "legendFormat": "Tables", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(chi_clickhouse_metric_NumberOfDatabases{chi=~\"$chi\",hostname=~\"$hostname\"})", + "hide": false, + "instant": false, + "legendFormat": "Databases", + "range": true, + "refId": "B" + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#265d1fd9", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "pattern": "(\\d\\d)(?:00(\\d)|0(\\d\\d)|(\\d\\d\\d))0*(.*)", + "result": { + "index": 0, + "text": "$1.$2$3$4.$5" + } + }, + "type": "regex" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 0 + }, + "hideTimeOverride": false, + "id": 62, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Version$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "editorMode": "code", + "exemplar": true, + "expr": "max(chi_clickhouse_metric_VersionInteger{chi=~\"$chi\",hostname=~\"$hostname\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Version", + "metric": "chi_clickhouse_metric_VersionInteger", + "range": true, + "refId": "A", + "step": 60 + } + ], + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "chi-(.*)\\.svc\\.cluster\\.local", + "renamePattern": "$1" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "type": "stat" }, { - "cacheTimeout": null, "colorBackground": false, "colorValue": true, "colors": [ @@ -322,12 +595,29 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Check Zookeeper connection, Disk Free space and network interconnection between replicas ASAP", "editable": true, "error": false, "fieldConfig": { - "defaults": {}, + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, "overrides": [] }, "format": "none", @@ -345,8 +635,6 @@ "y": 2 }, "id": 6, - "interval": null, - "isNew": true, "links": [ { "targetBlank": true, @@ -372,7 +660,23 @@ ], "maxDataPoints": 100, "nullPointMode": "connected", - "nullText": null, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.3", "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -394,7 +698,10 @@ "tableColumn": "", "targets": [ { - "expr": "sum(chi_clickhouse_metric_ReadonlyReplica{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(chi_clickhouse_metric_ReadonlyReplica{chi=~\"$chi\",hostname=~\"$hostname\"})", "intervalFactor": 2, "legendFormat": "", "refId": "A", @@ -403,7 +710,7 @@ ], "thresholds": "1,1", "title": "ReadOnly replicas", - "type": "singlestat", + "type": "stat", "valueFontSize": "80%", "valueMaps": [ { @@ -415,41 +722,72 @@ "valueName": "current" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Show DNS errors and distributed server-server connections failures", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 4 }, - "hiddenSeries": false, "id": 21, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideZero": false, - "max": true, - "min": true, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [ { "targetBlank": true, @@ -462,22 +800,30 @@ "url": "https://github.com/ClickHouse/ClickHouse/search?q=DNSError" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "increase(chi_clickhouse_event_NetworkErrors{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_NetworkErrors{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "NetworkErrors {{hostname}}", "metric": "chi_clickhouse_event_NetworkErrors", @@ -485,7 +831,10 @@ "step": 120 }, { - "expr": "increase(chi_clickhouse_event_DistributedConnectionFailAtAll{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DistributedConnectionFailAtAll{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "DistributedConnectionFailAtAll {{hostname}}", "metric": "chi_clickhouse_event_DistributedConnectionFailAtAll", @@ -493,7 +842,10 @@ "step": 120 }, { - "expr": "increase(chi_clickhouse_event_DistributedConnectionFailTry{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DistributedConnectionFailTry{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "DistributedConnectionFailTry {{hostname}}", "metric": "chi_clickhouse_event_DistributedConnectionFailTry", @@ -501,7 +853,10 @@ "step": 120 }, { - "expr": "increase(chi_clickhouse_event_DNSError{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DNSError{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "DNSErrors {{hostname}}", "metric": "chi_clickhouse_event_NetworkErrors", @@ -509,83 +864,76 @@ "step": 120 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "DNS and Distributed Connection Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Show readonly and partial shutdown replicas, zookeeer exceptions, zookeeer sessions, zookeeper init requests", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 4 }, - "hiddenSeries": false, "id": 19, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideZero": false, - "max": true, - "min": true, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [ { "targetBlank": true, @@ -603,22 +951,30 @@ "url": "https://www.slideshare.net/Altinity/introduction-to-the-mysteries-of-clickhouse-replication-by-robert-hodges-and-altinity-engineering-team" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_ReadonlyReplica{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReadonlyReplica{chi=~\"$chi\",hostname=~\"$hostname\"}", "hide": false, "intervalFactor": 2, "legendFormat": "ReadonlyReplica {{hostname}}", @@ -627,7 +983,10 @@ "step": 120 }, { - "expr": "increase(chi_clickhouse_event_ReplicaPartialShutdown{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ReplicaPartialShutdown{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "ReplicaPartialShutdown {{hostname}}", "metric": "chi_clickhouse_event_ReplicaPartialShutdown", @@ -635,7 +994,10 @@ "step": 120 }, { - "expr": "increase(chi_clickhouse_event_ZooKeeperUserExceptions{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperUserExceptions{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "hide": true, "intervalFactor": 2, "legendFormat": "ZooKeeperUserExceptions {{hostname}}", @@ -644,7 +1006,10 @@ "step": 120 }, { - "expr": "increase(chi_clickhouse_event_ZooKeeperInit{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperInit{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "ZooKeeperInit {{hostname}}", "metric": "chi_clickhouse_event_ZooKeeperInit", @@ -652,7 +1017,10 @@ "step": 120 }, { - "expr": "increase(chi_clickhouse_metric_ZooKeeperSession{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_metric_ZooKeeperSession{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "ZooKeeperSession {{hostname}}", "metric": "chi_clickhouse_metric_ZooKeeperSession", @@ -660,7 +1028,10 @@ "step": 120 }, { - "expr": "increase(chi_clickhouse_event_ZooKeeperHardwareExceptions{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperHardwareExceptions{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "ZooKeeperHardwareExceptions {{hostname}}", "metric": "chi_clickhouse_event_ZooKeeperUserExceptions", @@ -668,87 +1039,76 @@ "step": 120 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Replication and ZooKeeper Exceptions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "delayed query\nNumber of INSERT queries that are throttled due to high number of active data parts for partition in a *MergeTree table.\n\ndelayed blocks\nNumber of times the INSERT of a block to a *MergeTree table was throttled due to high number of active data parts for partition. \n\nrejected blocks\nNumber of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.\n\n\nplease look\nparts_to_delay_insert\nparts_to_throw_insert\n\nin system.merge_tree_settings table", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 4 }, - "hiddenSeries": false, "id": 5, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -761,134 +1121,293 @@ "url": "https://clickhouse.com/docs/en/operations/system-tables/merge_tree_settings" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_DelayedInserts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_DelayedInserts{chi=~\"$chi\",hostname=~\"$hostname\"}", "intervalFactor": 2, "legendFormat": "delayed queries {{hostname}}", "refId": "A", "step": 10 }, { - "expr": "increase(chi_clickhouse_event_DelayedInserts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DelayedInserts{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "delayed blocks {{hostname}}", "refId": "B", "step": 10 }, { - "expr": "increase(chi_clickhouse_event_RejectedInserts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_RejectedInserts{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "rejected blocks {{hostname}}", "refId": "C", "step": 10 }, { - "expr": "chi_clickhouse_metric_DistributedFilesToInsert{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_DistributedFilesToInsert{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "pending distributed files {{ hostname }}", "refId": "D" }, { - "expr": "chi_clickhouse_metric_BrokenDistributedFilesToInsert{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_BrokenDistributedFilesToInsert{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "broken distributed files {{ hostname }}", "refId": "E" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Delayed/Rejected/Pending Inserts", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "Number of SELECT queries started to be interpreted and maybe executed. Does not include queries that are failed to parse, that are rejected due to AST size limits; rejected due to quota limits or limits on number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.", - "editable": true, - "error": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of executing queries", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisWidth": 55, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 29, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineWidth": 0, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Overall" + }, + "properties": [ + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 11 }, - "hiddenSeries": false, - "id": 1, - "isNew": true, - "legend": { - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": false, - "total": false, - "values": true + "id": 63, + "links": [ + { + "targetBlank": true, + "title": "max_concurent_queries", + "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max_concurrent_queries" + }, + { + "targetBlank": true, + "title": "max_execution_time", + "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "lines": true, - "linewidth": 2, + "pluginVersion": "8.3.2", + "targets": [ + { + "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "editorMode": "code", + "exemplar": true, + "expr": "max by (hostname) (max_over_time(chi_clickhouse_metric_Query{chi=~\"$chi\",hostname=~\"$hostname\"}[$__interval])-1) OR on () vector(0) > 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{hostname}}", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "editorMode": "code", + "exemplar": true, + "expr": "sum(chi_clickhouse_metric_Query{chi=~\"$chi\",hostname=~\"$hostname\"}-1) OR on () vector(0)", + "hide": false, + "interval": "", + "intervalFactor": 5, + "legendFormat": "Overall", + "range": true, + "refId": "Overall", + "step": 10 + } + ], + "title": "Queries (running)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of executing select queries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisWidth": 55, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 8, "links": [ { "targetBlank": true, @@ -901,271 +1420,105 @@ "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "rate(chi_clickhouse_event_SelectQuery{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(chi_clickhouse_event_SelectQuery{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])) OR on () vector(0)", + "hide": false, "intervalFactor": 2, - "legendFormat": "select {{hostname}}", - "refId": "B", - "step": 10 - }, - { - "expr": "rate(chi_clickhouse_event_Query{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": true, - "intervalFactor": 2, - "legendFormat": "total {{hostname}}", + "legendFormat": "Select", + "range": true, "refId": "A", "step": 10 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Select Queries", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Select Queries (started per sec)", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "Show how much bytes read and decompress via compressed buffer on each server in cluster", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] + "datasource": { + "uid": "${DS_PROMETHEUS}" }, - "fill": 1, - "fillGradient": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 11 - }, - "hiddenSeries": false, - "id": 8, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - { - "targetBlank": true, - "title": "I/O buffers architecture", - "url": "https://clickhouse.com/docs/en/development/architecture/#io" - } - ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/^uncompressed.+/", - "color": "#73BF69" - }, - { - "alias": "/^(file descriptor|os).+/", - "color": "#F2495C" - }, - { - "alias": "/^compressed.+/", - "color": "#FADE2A" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(chi_clickhouse_event_CompressedReadBufferBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": false, - "intervalFactor": 2, - "legendFormat": "uncompressed {{hostname}}", - "refId": "A", - "step": 10 - }, - { - "expr": "rate(chi_clickhouse_event_ReadCompressedBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "compressed {{hostname}}", - "refId": "C", - "step": 10 - }, - { - "expr": "rate(chi_clickhouse_event_ReadBufferFromFileDescriptorReadBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "file descriptor {{hostname}}", - "refId": "B", - "step": 10 - }, - { - "expr": "rate(chi_clickhouse_event_OSReadBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "os {{hostname}}", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Read Bytes", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 10, - "max": null, - "min": "0", - "show": true - }, - { - "decimals": null, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, "description": "Total amount of memory (bytes) allocated in currently executing queries. \n\nNote that some memory allocations may not be accounted.", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 11 }, - "hiddenSeries": false, "id": 13, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -1173,110 +1526,101 @@ "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_MemoryTracking{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryTracking{chi=~\"$chi\",hostname=~\"$hostname\"}", "intervalFactor": 2, "legendFormat": "{{hostname}}", "refId": "A", "step": 10 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Memory for Queries", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "Number of INSERT queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.", - "editable": true, - "error": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of running INSERT queries. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 18 }, - "hiddenSeries": false, "id": 30, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -1284,223 +1628,211 @@ "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - {}, - {} - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "rate(chi_clickhouse_event_InsertQuery{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "irate(chi_clickhouse_event_InsertQuery{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "legendFormat": "Insert queries {{hostname}}", + "range": true, "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Insert Queries", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "reqps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Insert Queries (running)", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "## Tracks amount of inserted data.", - "editable": true, - "error": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of executing insert queries", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisWidth": 55, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 18 }, - "hiddenSeries": false, - "id": 37, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, + "id": 58, "links": [ { "targetBlank": true, - "title": "max_memory_usage", - "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage" - } - ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - {}, - {} - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(chi_clickhouse_event_InsertedBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Insert bytes {{hostname}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Bytes Inserted", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true + "title": "max_concurent_queries", + "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max_concurrent_queries" }, { - "format": "reqps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "targetBlank": true, + "title": "max_execution_time", + "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time" } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(chi_clickhouse_event_InsertQuery{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])) OR on () vector(0)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Select", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Insert Queries (started per sec)", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "## Tracks rows of inserted data.", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 18 }, - "hiddenSeries": false, "id": 32, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -1508,112 +1840,174 @@ "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - {}, - {} - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "rate(chi_clickhouse_event_InsertedRows{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_InsertedRows{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "legendFormat": "Insert rows {{hostname}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Rows Inserted", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "reqps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Show how intensive data exchange between replicas in parts", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^max.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFA6B0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^check.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^fetch.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B877D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^(data loss|fetch fail|check fail).+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^replicated merge.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#DEB6F2", + "mode": "fixed" + } + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 25 }, - "hiddenSeries": false, "id": 3, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -1621,49 +2015,29 @@ "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:227", - "alias": "/^max.+/", - "color": "#FFA6B0" + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - { - "$$hashKey": "object:228", - "alias": "/^check.+/", - "color": "#FF9830" - }, - { - "$$hashKey": "object:229", - "alias": "/^fetch.+/", - "color": "#B877D9" - }, - { - "$$hashKey": "object:338", - "alias": "/^(data loss|fetch fail|check fail).+/", - "color": "#C4162A" - }, - { - "$$hashKey": "object:1252", - "alias": "/^replicated merge.+/", - "color": "#DEB6F2" + "tooltip": { + "mode": "multi", + "sort": "none" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "10.4.3", "targets": [ { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "exemplar": true, - "expr": "rate(chi_clickhouse_event_ReplicatedDataLoss{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "expr": "irate(chi_clickhouse_event_ReplicatedDataLoss{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "interval": "", "intervalFactor": 2, "legendFormat": "data loss {{hostname}}", @@ -1671,8 +2045,11 @@ "step": 20 }, { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "exemplar": true, - "expr": "rate(chi_clickhouse_event_ReplicatedPartChecks{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "expr": "irate(chi_clickhouse_event_ReplicatedPartChecks{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "hide": false, "interval": "", "intervalFactor": 2, @@ -1681,8 +2058,11 @@ "step": 20 }, { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "exemplar": true, - "expr": "rate(chi_clickhouse_event_ReplicatedPartChecksFailed{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "expr": "irate(chi_clickhouse_event_ReplicatedPartChecksFailed{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "hide": false, "interval": "", "intervalFactor": 2, @@ -1691,8 +2071,11 @@ "step": 20 }, { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "exemplar": true, - "expr": "rate(chi_clickhouse_event_ReplicatedPartFetches{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "expr": "irate(chi_clickhouse_event_ReplicatedPartFetches{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "hide": false, "interval": "", "intervalFactor": 2, @@ -1701,8 +2084,11 @@ "step": 20 }, { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "exemplar": true, - "expr": "rate(chi_clickhouse_event_ReplicatedPartFailedFetches{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "expr": "irate(chi_clickhouse_event_ReplicatedPartFailedFetches{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "hide": false, "interval": "", "intervalFactor": 2, @@ -1711,8 +2097,11 @@ "step": 20 }, { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "exemplar": true, - "expr": "rate(chi_clickhouse_event_ReplicatedPartFetchesOfMerged{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "expr": "irate(chi_clickhouse_event_ReplicatedPartFetchesOfMerged{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "hide": false, "interval": "", "intervalFactor": 2, @@ -1721,8 +2110,11 @@ "step": 20 }, { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "exemplar": true, - "expr": "rate(chi_clickhouse_event_ReplicatedPartMerges{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "expr": "irate(chi_clickhouse_event_ReplicatedPartMerges{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "hide": false, "interval": "", "intervalFactor": 2, @@ -1731,98 +2123,122 @@ "step": 20 }, { - "expr": "chi_clickhouse_metric_ReplicasSumInsertsInQueue{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReplicasSumInsertsInQueue{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "inserts in queue {{hostname}}", "refId": "H" }, { - "expr": "chi_clickhouse_metric_ReplicasSumMergesInQueue{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReplicasSumMergesInQueue{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "merges in queue {{hostname}}", "refId": "I" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Replication Queue Jobs", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Show seconds when replicated servers can be delayed relative to current time, when you insert directly in *ReplicatedMegreTree table on one server clickhouse need time to replicate new parts of data to another servers in same shard in background", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^absolute.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^relative.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FADE2A", + "mode": "fixed" + } + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 25 }, - "hiddenSeries": false, - "id": 7, - "isNew": true, - "legend": { - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, + "id": 59, "links": [ { "targetBlank": true, @@ -1840,126 +2256,113 @@ "url": "https://clickhouse.com/docs/en/operations/settings/settings#settings-max_replica_delay_for_distributed_queries" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/^absolute.+/", - "color": "#F2495C" + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - { - "alias": "/^relative.+/", - "color": "#FADE2A" + "tooltip": { + "mode": "multi", + "sort": "none" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_ReplicasMaxAbsoluteDelay{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReplicasMaxAbsoluteDelay{chi=~\"$chi\",hostname=~\"$hostname\"}", "intervalFactor": 2, "legendFormat": "absolute {{hostname}}", "refId": "A", "step": 10 }, { - "expr": "chi_clickhouse_metric_ReplicasMaxRelativeDelay{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReplicasMaxRelativeDelay{chi=~\"$chi\",hostname=~\"$hostname\"}", "intervalFactor": 2, "legendFormat": "relative {{hostname}}", "refId": "B", "step": 10 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Max Replica Delay", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Number of requests to ZooKeeper transactions per seconds.", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 25 }, - "hiddenSeries": false, "id": 34, - "isNew": true, - "legend": { - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -1967,114 +2370,110 @@ "url": "https://clickhouse.com/docs/en/development/architecture#replication" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "rate(chi_clickhouse_event_ZooKeeperTransactions{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_ZooKeeperTransactions{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "legendFormat": "transactions {{ hostname }}", "refId": "B" }, { - "expr": "chi_clickhouse_metric_ZooKeeperRequest{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ZooKeeperRequest{chi=~\"$chi\",hostname=~\"$hostname\"}", "hide": true, "legendFormat": "{{ hostname }}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Zookeeper Transactions", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Show how intensive background merge processes", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 32 }, - "hiddenSeries": false, "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -2087,111 +2486,102 @@ "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "rate(chi_clickhouse_event_Merge{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_Merge{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "merges {{hostname}}", "refId": "A", "step": 4 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Merges", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Show how intensive background merge processes", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 32 }, - "hiddenSeries": false, "id": 36, - "isNew": true, - "legend": { - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -2204,111 +2594,102 @@ "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "rate(chi_clickhouse_event_MergedRows{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_MergedRows{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "rows {{hostname}}", "refId": "B", "step": 4 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Merged Rows", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Show how intensive background merge processes", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 32 }, - "hiddenSeries": false, "id": 49, - "isNew": true, - "legend": { - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -2321,109 +2702,102 @@ "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "rate(chi_clickhouse_event_MergedUncompressedBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_MergedUncompressedBytes{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", "intervalFactor": 2, "legendFormat": "bytes {{hostname}}", "refId": "B", "step": 4 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Merged Uncompressed Bytes", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "decbytes", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "decimals": 0, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 39 }, - "hiddenSeries": false, "id": 23, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": true, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [ { "targetBlank": true, @@ -2436,105 +2810,161 @@ "url": "https://github.com/ClickHouse/ClickHouse/search?q=parts_to_delay_insert" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "sum by(hostname) (chi_clickhouse_table_parts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\",active=\"1\"})", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by(hostname) (chi_clickhouse_table_parts{chi=~\"$chi\",hostname=~\"$hostname\",active=\"1\"})", "legendFormat": "Parts {{hostname}}", "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Active Parts", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "decimals": 0, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*detached_by_user.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*broken.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*(clone|ignored).*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Inactive/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 39 }, - "hiddenSeries": false, "id": 50, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": true, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [ { "targetBlank": true, @@ -2542,136 +2972,114 @@ "url": "https://clickhouse.com/docs/en/operations/system-tables/detached_parts/" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:254", - "alias": "/.*detached_by_user.*/", - "color": "#CA95E5" + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - { - "$$hashKey": "object:285", - "alias": "/.*broken.*/", - "color": "#E02F44" - }, - { - "$$hashKey": "object:355", - "alias": "/.*(clone|ignored).*/", - "color": "#FFEE52" - }, - { - "$$hashKey": "object:447", - "alias": "/^Inactive/", - "yaxis": 2 + "tooltip": { + "mode": "multi", + "sort": "none" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "sum by(hostname,reason) (chi_clickhouse_metric_DetachedParts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by(hostname,reason) (chi_clickhouse_metric_DetachedParts{chi=~\"$chi\",hostname=~\"$hostname\"})", "interval": "", "legendFormat": "{{reason}} {{hostname}} ", "refId": "C" }, { - "expr": "sum by(hostname) (chi_clickhouse_table_parts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\",active=\"0\"})", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by(hostname) (chi_clickhouse_table_parts{chi=~\"$chi\",hostname=~\"$hostname\",active=\"0\"})", "hide": true, "interval": "", "legendFormat": "Inactive {{hostname}} ", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Detached parts", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": false - } - ], - "yaxis": { - "align": true, - "alignLevel": 0 - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Each logical partition defined over `PARTITION BY` contains few physical data \"parts\" ", - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 39 }, - "hiddenSeries": false, "id": 4, - "isNew": true, - "legend": { - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, "links": [ { "targetBlank": true, @@ -2689,104 +3097,178 @@ "url": "https://clickhouse.com/docs/en/operations/system-tables/part-log" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_MaxPartCountForPartition{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MaxPartCountForPartition{chi=~\"$chi\",hostname=~\"$hostname\"}", "intervalFactor": 2, "legendFormat": "{{hostname}}", "refId": "A", "step": 10 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Max Part count for Partition", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Memory size allocated for clickhouse-server process\nAvailable for ClickHouse 20.4+\n\nVIRT \nThe total amount of virtual memory used by the task. It includes all code, data and shared libraries plus pages that have been swapped out.\n\nVIRT = SWAP + RES\n\n\nSWAP -- Swapped size (kb)\nThe swapped out portion of a task's total virtual memory image.\n\nRES -- Resident size (kb)\nThe non-swapped physical memory a task has used.\nRES = CODE + USED DATA.\n\nCODE -- Code size (kb)\nThe amount of physical memory devoted to executable code, also known as the 'text resident set' size or TRS\n\nDATA -- Data+Stack size (kb)\nThe amount of physical memory allocated to other than executable code, also known as the 'data resident set' size or DRS.\n\nSHR -- Shared Mem size (kb)\nThe amount of shared memory used by a task. It simply reflects memory that could be potentially shared with other processes.", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/VIRT.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/DATA.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/CODE.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/RES.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FADE2A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/SHR.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + } + ] + } + ] }, - "fill": 1, - "fillGradient": 2, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 46 }, - "hiddenSeries": false, "id": 46, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ { "targetBlank": true, @@ -2794,142 +3276,130 @@ "url": "https://elinux.org/Runtime_Memory_Measurement" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/VIRT.+/", - "color": "#73BF69" + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - { - "alias": "/DATA.+/", - "color": "#C4162A" - }, - { - "alias": "/CODE.+/", - "color": "#FF9830" - }, - { - "alias": "/RES.+/", - "color": "#FADE2A" - }, - { - "alias": "/SHR.+/", - "color": "#5794F2" + "tooltip": { + "mode": "multi", + "sort": "none" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": true, + }, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_MemoryCode{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryCode{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "CODE {{ hostname }}", "refId": "A" }, { - "expr": "chi_clickhouse_metric_MemoryResident{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryResident{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "RES {{ hostname }}", "refId": "B" }, { - "expr": "chi_clickhouse_metric_MemoryShared{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryShared{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "SHR {{ hostname }}", "refId": "C" }, { - "expr": "chi_clickhouse_metric_MemoryDataAndStack{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryDataAndStack{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "DATA {{ hostname }}", "refId": "D" }, { - "expr": "chi_clickhouse_metric_MemoryVirtual{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryVirtual{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "VIRT {{ hostname }}", "refId": "E" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": " clickhouse-server Process Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Memory size allocated for primary keys", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 46 }, - "hiddenSeries": false, "id": 45, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ { "targetBlank": true, @@ -2937,101 +3407,98 @@ "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#selecting-the-primary-key" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_MemoryPrimaryKeyBytesAllocated{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryPrimaryKeyBytesAllocated{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "{{ hostname }}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Primary Keys Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "Memory size allocated for dictionaries", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 46 }, - "hiddenSeries": false, "id": 43, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ { "targetBlank": true, @@ -3044,101 +3511,99 @@ "url": "https://clickhouse.com/docs/en/sql-reference/statements/create/dictionary" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_MemoryDictionaryBytesAllocated{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryDictionaryBytesAllocated{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "{{ hostname }}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Dictionary Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "description": "shows how much space available in the kubernetes pod\n\nbe careful with multiple volumes configuration, kubernetes volume claims and S3 as storage backend", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 53 }, - "hiddenSeries": false, "id": 39, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ { "targetBlank": true, @@ -3151,102 +3616,330 @@ "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-multiple-volumes" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_DiskFreeBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"} / chi_clickhouse_metric_DiskTotalBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_DiskFreeBytes{chi=~\"$chi\",hostname=~\"$hostname\"} / chi_clickhouse_metric_DiskTotalBytes{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "{{ disk }} {{hostname}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Disk Space Free", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "Total data size for all ClickHouse *MergeTree tables\n\n", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Bytes" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + }, + { + "id": "custom.width", + "value": 233 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Rows" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.width", + "value": 118 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "database" + }, + "properties": [ + { + "id": "custom.width", + "value": 199 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "table" + }, + "properties": [ + { + "id": "custom.width", + "value": 238 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Parts" + }, + "properties": [ + { + "id": "custom.width", + "value": 101 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "BytePerRow" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 7, - "w": 8, + "h": 14, + "w": 16, "x": 8, "y": 53 }, - "hiddenSeries": false, - "id": 41, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "id": 61, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "frameIndex": 2, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Bytes" + } + ] }, - "lines": true, - "linewidth": 1, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (database, table) (chi_clickhouse_table_parts_bytes{chi=~\"$chi\",hostname=~\"$hostname\", active=\"1\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Bytes", + "refId": "Bytes" + }, + { + "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (database, table) (chi_clickhouse_table_parts_rows{chi=~\"$chi\",hostname=~\"$hostname\", active=\"1\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Rows", + "refId": "Rows" + }, + { + "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (database, table) (chi_clickhouse_table_parts{chi=~\"$chi\",hostname=~\"$hostname\", active=\"1\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Parts", + "refId": "Parts" + } + ], + "title": "Table Stats", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": {}, + "renameByName": { + "Value #Bytes": "Bytes", + "Value #Parts": "Parts", + "Value #Rows": "Rows" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "BytePerRow", + "binary": { + "left": "Bytes", + "operator": "/", + "reducer": "sum", + "right": "Rows" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total data size for all ClickHouse *MergeTree tables\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 60 + }, + "id": 41, "links": [ { "targetBlank": true, @@ -3254,107 +3947,1390 @@ "url": "https://clickhouse.com/docs/en/operations/system-tables/parts" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_DiskDataBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_DiskDataBytes{chi=~\"$chi\",hostname=~\"$hostname\"}", "legendFormat": "{{ hostname }}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Clickhouse Data size on Disk", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "Show different types of connections for each server", - "editable": true, - "error": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "BackgroundPoolTask\t\n---\nNumber of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping)\n\n\nBackgroundMovePoolTask\n---\nNumber of active tasks in BackgroundProcessingPool for moves\n\n\nBackgroundSchedulePoolTask\t\n---\nA number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 67 + }, + "id": 9, + "links": [ + { + "targetBlank": true, + "title": "FETCH PARTITION", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter/partition#fetch-partitionpart" + }, + { + "targetBlank": true, + "title": "Mutations of data", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter#mutations" + }, + { + "targetBlank": true, + "title": "Data TTL", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl" + }, + { + "targetBlank": true, + "title": "MOVE PARTITION", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter/partition#move-partitionpart" + } + ], + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_BackgroundPoolTask{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "merge, mutate, fetch {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_BackgroundSchedulePoolTask{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "clean, alter, replica re-init {{hostname}}", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_BackgroundMovePoolTask{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "moves {{hostname}}", + "refId": "C", + "step": 10 + } + ], + "title": "Background Tasks", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of active mutations (ALTER DELETE/ALTER UPDATE) and parts to mutate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 67 + }, + "id": 26, + "links": [ + { + "targetBlank": true, + "title": "Mutations", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter#mutations" + }, + { + "targetBlank": true, + "title": "system.mutations", + "url": "https://clickhouse.com/docs/en/operations/system-tables/mutations" + }, + { + "targetBlank": true, + "title": "KILL MUTATION", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/kill#kill-mutation" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (hostname) (chi_clickhouse_table_mutations{chi=~\"$chi\",hostname=~\"$hostname\"})", + "legendFormat": "mutations {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (hostname) (chi_clickhouse_table_mutations_parts_to_do{chi=~\"$chi\",hostname=~\"$hostname\"})", + "legendFormat": "parts_to_do {{hostname}}", + "refId": "B" + } + ], + "title": "Mutations", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show which percent of mark files (.mrk) read from memory instead of disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, - "y": 53 + "y": 67 }, - "hiddenSeries": false, - "id": 48, - "isNew": true, + "id": 11, + "links": [ + { + "targetBlank": true, + "title": "mark_cache_size", + "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#server-mark-cache-size" + }, + { + "targetBlank": true, + "title": "MergeTree architecture", + "url": "https://clickhouse.com/docs/en/development/architecture/#merge-tree" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_MarkCacheHits{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) / (irate(chi_clickhouse_event_MarkCacheHits{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) + irate(chi_clickhouse_event_MarkCacheMisses{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "refId": "A", + "step": 4 + } + ], + "title": "Marks Cache Hit Rate", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "The time which CPU spent on various types of activity ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Disk Read.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Disk Write.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0B400", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Real Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^User Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFF899", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^System Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS IO Wait.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS CPU Wait.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgb(95, 29, 29)", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS CPU Virtual.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B877D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Network Receive.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C0D8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Network Send.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 74 + }, + "id": 51, + "interval": "", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_DiskReadElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Disk Read syscall {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Disk Write syscall {{hostname}}", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Network Receive {{hostname}}", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Network Send {{hostname}}", + "refId": "D" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_RealTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Real Time {{hostname}}", + "refId": "E" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_UserTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "User Time {{hostname}}", + "refId": "F" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_SystemTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "System Time {{hostname}}", + "refId": "G" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_OSIOWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS IO Wait {{hostname}}", + "refId": "H" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_OSCPUWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Wait {{hostname}}", + "refId": "I" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Virtual {{hostname}}", + "refId": "J" + } + ], + "title": "CPU Time per second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "The time which CPU spent on various types of activity ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Disk Read.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Disk Write.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0B400", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Real Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^User Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFF899", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^System Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS IO Wait.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS CPU Wait.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgb(95, 29, 29)", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS CPU Virtual.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B877D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Network Receive.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C0D8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Network Send.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 74 + }, + "id": 54, + "interval": "", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_DiskReadElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "legendFormat": "Disk Read syscall {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "legendFormat": "Disk Write syscall {{hostname}}", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "legendFormat": "Network Receive {{hostname}}", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "legendFormat": "Network Send {{hostname}}", + "refId": "D" + } + ], + "title": "Network / Disk CPU Time per second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 74 + }, + "id": 55, + "interval": "", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "chi_clickhouse_metric_LoadAverage1{chi=~\"$chi\",hostname=~\"$hostname\"}", + "hide": false, + "interval": "", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "title": "Load Average 1m", + "type": "timeseries" + }, + { + "aliasColors": {}, + "breakPoint": "50%", + "combine": { + "label": "Others", + "threshold": "0.01" + }, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "The time which CPU spent on various types of activity total for the selected period", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "µs" + }, + "overrides": [] + }, + "fontSize": "80%", + "format": "µs", + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 81 + }, + "id": 52, + "interval": "1m", "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "header": "", + "percentage": true, + "show": true, + "sort": "total", + "sortDesc": true, + "values": true }, - "lines": true, - "linewidth": 2, + "legendType": "Right side", + "nullPointMode": "connected", + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pieType": "pie", + "strokeWidth": "", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DiskReadElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Disk Read syscall {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DiskWriteElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Disk Write syscall {{hostname}}", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Network Receive {{hostname}}", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_NetworkSendElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Network Send {{hostname}}", + "refId": "D" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_RealTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Real Time {{hostname}}", + "refId": "E" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_UserTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "User Time {{hostname}}", + "refId": "F" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_SystemTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "System Time {{hostname}}", + "refId": "G" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_OSIOWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS IO Wait {{hostname}}", + "refId": "H" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_OSCPUWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Wait {{hostname}}", + "refId": "I" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Virtual {{hostname}}", + "refId": "J" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ThrottlerSleepMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Throttler Sleep {{hostname}}", + "refId": "K" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DelayedInsertsMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "Delayed Insert {{hostname}}", + "refId": "L" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Zookeeper Wait {{hostname}}", + "refId": "M" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_CompileExpressionsMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Compile Expressions {{hostname}}", + "refId": "N" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_MergesTimeMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "Merges {{hostname}}", + "refId": "O" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_RWLockReadersWaitMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "RWLock Reader Wait {{hostname}}", + "refId": "P" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_RWLockWritersWaitMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "RWLock Writer Wait {{hostname}}", + "refId": "Q" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_SelectQueryTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Select Query {{hostname}}", + "refId": "R" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_InsertQueryTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Insert Query {{hostname}}", + "refId": "S" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_S3ReadMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "S3 Read {{hostname}}", + "refId": "T" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_S3WriteMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "S3 Write {{hostname}}", + "refId": "U" + } + ], + "title": "CPU Time total", + "type": "piechart", + "valueName": "total" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show different types of connections for each server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 81 + }, + "id": 48, "links": [ { "targetBlank": true, @@ -3382,1188 +5358,69 @@ "url": "https://clickhouse.com/docs/en/interfaces/tcp/" } ], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, + "pluginVersion": "10.4.3", "targets": [ { - "expr": "chi_clickhouse_metric_TCPConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_TCPConnection{chi=~\"$chi\",hostname=~\"$hostname\"}", "intervalFactor": 2, "legendFormat": "tcp {{hostname}}", "refId": "A", "step": 10 }, { - "expr": "chi_clickhouse_metric_HTTPConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_HTTPConnection{chi=~\"$chi\",hostname=~\"$hostname\"}", "intervalFactor": 2, "legendFormat": "http {{hostname}}", "refId": "B", "step": 10 }, { - "expr": "chi_clickhouse_metric_InterserverConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_InterserverConnection{chi=~\"$chi\",hostname=~\"$hostname\"}", "intervalFactor": 2, "legendFormat": "interserver {{hostname}}", "refId": "C", "step": 10 }, { - "expr": "chi_clickhouse_metric_MySQLConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MySQLConnection{chi=~\"$chi\",hostname=~\"$hostname\"}", "intervalFactor": 2, "legendFormat": "mysql {{hostname}}", "refId": "D", "step": 10 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Connections", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "BackgroundPoolTask\t\n---\nNumber of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping)\n\n\nBackgroundMovePoolTask\n---\nNumber of active tasks in BackgroundProcessingPool for moves\n\n\nBackgroundSchedulePoolTask\t\n---\nA number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 60 - }, - "hiddenSeries": false, - "id": 9, - "isNew": true, - "legend": { - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [ - { - "targetBlank": true, - "title": "FETCH PARTITION", - "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter/partition#fetch-partitionpart" - }, - { - "targetBlank": true, - "title": "Mutations of data", - "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter#mutations" - }, - { - "targetBlank": true, - "title": "Data TTL", - "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl" - }, - { - "targetBlank": true, - "title": "MOVE PARTITION", - "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter/partition#move-partitionpart" - } - ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "chi_clickhouse_metric_BackgroundPoolTask{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", - "intervalFactor": 2, - "legendFormat": "merge, mutate, fetch {{hostname}}", - "refId": "A", - "step": 10 - }, - { - "expr": "chi_clickhouse_metric_BackgroundSchedulePoolTask{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", - "intervalFactor": 2, - "legendFormat": "clean, alter, replica re-init {{hostname}}", - "refId": "B", - "step": 10 - }, - { - "expr": "chi_clickhouse_metric_BackgroundMovePoolTask{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", - "intervalFactor": 2, - "legendFormat": "moves {{hostname}}", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Background Tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "Number of active mutations (ALTER DELETE/ALTER UPDATE) and parts to mutate", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 60 - }, - "hiddenSeries": false, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - { - "targetBlank": true, - "title": "Mutations", - "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter#mutations" - }, - { - "targetBlank": true, - "title": "system.mutations", - "url": "https://clickhouse.com/docs/en/operations/system-tables/mutations/" - }, - { - "targetBlank": true, - "title": "KILL MUTATION", - "url": "https://clickhouse.com/docs/en/sql-reference/statements/kill#kill-mutation" - } - ], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum by (hostname) (chi_clickhouse_table_mutations{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})", - "legendFormat": "mutations {{hostname}}", - "refId": "A" - }, - { - "expr": "sum by (hostname) (chi_clickhouse_table_mutations_parts_to_do{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})", - "legendFormat": "parts_to_do {{hostname}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Mutations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "Show which percent of mark files (.mrk) read from memory instead of disk", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 60 - }, - "hiddenSeries": false, - "id": 11, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - { - "targetBlank": true, - "title": "mark_cache_size", - "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#server-mark-cache-size" - }, - { - "targetBlank": true, - "title": "MergeTree architecture", - "url": "https://clickhouse.com/docs/en/development/architecture/#merge-tree" - } - ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(chi_clickhouse_event_MarkCacheHits{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) / (rate(chi_clickhouse_event_MarkCacheHits{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) + rate(chi_clickhouse_event_MarkCacheMisses{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{hostname}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Marks Cache Hit Rate", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "The time which CPU spent on various types of activity ", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 67 - }, - "hiddenSeries": false, - "id": 51, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/^Disk Read.+/", - "color": "#FF9830" - }, - { - "alias": "/^Disk Write.+/", - "color": "#E0B400" - }, - { - "alias": "/^Real Time.+/", - "color": "#73BF69" - }, - { - "alias": "/^User Time.+/", - "color": "#FFF899" - }, - { - "alias": "/^System Time.+/", - "color": "#F2495C" - }, - { - "alias": "/^OS IO Wait.+/", - "color": "#C4162A" - }, - { - "alias": "/^OS CPU Wait.+/", - "color": "rgb(95, 29, 29)" - }, - { - "alias": "/^OS CPU Virtual.+/", - "color": "#B877D9" - }, - { - "alias": "/^Network Receive.+/", - "color": "#C0D8FF" - }, - { - "alias": "/^Network Send.+/", - "color": "#8AB8FF" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "rate(chi_clickhouse_event_DiskReadElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": true, - "legendFormat": "Disk Read syscall {{hostname}}", - "refId": "A" - }, - { - "expr": "rate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": true, - "legendFormat": "Disk Write syscall {{hostname}}", - "refId": "B" - }, - { - "expr": "rate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": true, - "legendFormat": "Network Receive {{hostname}}", - "refId": "C" - }, - { - "expr": "rate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": true, - "legendFormat": "Network Send {{hostname}}", - "refId": "D" - }, - { - "expr": "rate(chi_clickhouse_event_RealTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Real Time {{hostname}}", - "refId": "E" - }, - { - "expr": "rate(chi_clickhouse_event_UserTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "User Time {{hostname}}", - "refId": "F" - }, - { - "expr": "rate(chi_clickhouse_event_SystemTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "System Time {{hostname}}", - "refId": "G" - }, - { - "expr": "rate(chi_clickhouse_event_OSIOWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "OS IO Wait {{hostname}}", - "refId": "H" - }, - { - "expr": "rate(chi_clickhouse_event_OSCPUWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "OS CPU Wait {{hostname}}", - "refId": "I" - }, - { - "expr": "rate(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "OS CPU Virtual {{hostname}}", - "refId": "J" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Time per second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "breakPoint": "50%", - "cacheTimeout": null, - "combine": { - "label": "Others", - "threshold": "0.01" - }, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "The time which CPU spent on various types of activity total for the selected period", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fontSize": "80%", - "format": "µs", - "gridPos": { - "h": 7, - "w": 16, - "x": 8, - "y": 67 - }, - "id": 52, - "interval": "1m", - "legend": { - "header": "", - "percentage": true, - "show": true, - "sideWidth": null, - "sort": "total", - "sortDesc": true, - "values": true - }, - "legendType": "Right side", - "links": [], - "nullPointMode": "connected", - "pieType": "pie", - "strokeWidth": "", - "targets": [ - { - "expr": "increase(chi_clickhouse_event_DiskReadElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Disk Read syscall {{hostname}}", - "refId": "A" - }, - { - "expr": "increase(chi_clickhouse_event_DiskWriteElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Disk Write syscall {{hostname}}", - "refId": "B" - }, - { - "expr": "increase(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Network Receive {{hostname}}", - "refId": "C" - }, - { - "expr": "increase(chi_clickhouse_event_NetworkSendElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Network Send {{hostname}}", - "refId": "D" - }, - { - "expr": "increase(chi_clickhouse_event_RealTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Real Time {{hostname}}", - "refId": "E" - }, - { - "expr": "increase(chi_clickhouse_event_UserTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "User Time {{hostname}}", - "refId": "F" - }, - { - "expr": "increase(chi_clickhouse_event_SystemTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "System Time {{hostname}}", - "refId": "G" - }, - { - "expr": "increase(chi_clickhouse_event_OSIOWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "OS IO Wait {{hostname}}", - "refId": "H" - }, - { - "expr": "increase(chi_clickhouse_event_OSCPUWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "OS CPU Wait {{hostname}}", - "refId": "I" - }, - { - "expr": "increase(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "OS CPU Virtual {{hostname}}", - "refId": "J" - }, - { - "expr": "increase(chi_clickhouse_event_ThrottlerSleepMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Throttler Sleep {{hostname}}", - "refId": "K" - }, - { - "expr": "increase(chi_clickhouse_event_DelayedInsertsMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", - "legendFormat": "Delayed Insert {{hostname}}", - "refId": "L" - }, - { - "expr": "increase(chi_clickhouse_event_ZooKeeperWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Zookeeper Wait {{hostname}}", - "refId": "M" - }, - { - "expr": "increase(chi_clickhouse_event_CompileExpressionsMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Compile Expressions {{hostname}}", - "refId": "N" - }, - { - "expr": "increase(chi_clickhouse_event_MergesTimeMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", - "legendFormat": "Merges {{hostname}}", - "refId": "O" - }, - { - "expr": "increase(chi_clickhouse_event_RWLockReadersWaitMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", - "legendFormat": "RWLock Reader Wait {{hostname}}", - "refId": "P" - }, - { - "expr": "increase(chi_clickhouse_event_RWLockWritersWaitMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", - "legendFormat": "RWLock Writer Wait {{hostname}}", - "refId": "Q" - }, - { - "expr": "increase(chi_clickhouse_event_SelectQueryTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Select Query {{hostname}}", - "refId": "R" - }, - { - "expr": "increase(chi_clickhouse_event_InsertQueryTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "Insert Query {{hostname}}", - "refId": "S" - }, - { - "expr": "increase(chi_clickhouse_event_S3ReadMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "S3 Read {{hostname}}", - "refId": "T" - }, - { - "expr": "increase(chi_clickhouse_event_S3WriteMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "legendFormat": "S3 Write {{hostname}}", - "refId": "U" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Time total", - "type": "piechart", - "valueName": "total" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "The time which CPU spent on various types of activity ", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 74 - }, - "hiddenSeries": false, - "id": 54, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/^Disk Read.+/", - "color": "#FF9830" - }, - { - "alias": "/^Disk Write.+/", - "color": "#E0B400" - }, - { - "alias": "/^Real Time.+/", - "color": "#73BF69" - }, - { - "alias": "/^User Time.+/", - "color": "#FFF899" - }, - { - "alias": "/^System Time.+/", - "color": "#F2495C" - }, - { - "alias": "/^OS IO Wait.+/", - "color": "#C4162A" - }, - { - "alias": "/^OS CPU Wait.+/", - "color": "rgb(95, 29, 29)" - }, - { - "alias": "/^OS CPU Virtual.+/", - "color": "#B877D9" - }, - { - "alias": "/^Network Receive.+/", - "color": "#C0D8FF" - }, - { - "alias": "/^Network Send.+/", - "color": "#8AB8FF" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "rate(chi_clickhouse_event_DiskReadElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": false, - "legendFormat": "Disk Read syscall {{hostname}}", - "refId": "A" - }, - { - "expr": "rate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": false, - "legendFormat": "Disk Write syscall {{hostname}}", - "refId": "B" - }, - { - "expr": "rate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": false, - "legendFormat": "Network Receive {{hostname}}", - "refId": "C" - }, - { - "expr": "rate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", - "hide": false, - "legendFormat": "Network Send {{hostname}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Network / Disk CPU Time per second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:193", - "format": "µs", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:194", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 74 - }, - "hiddenSeries": false, - "id": 24, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - { - "targetBlank": true, - "title": "Howto show detail statistic on grafana for golang process", - "url": "https://grafana.com/grafana/dashboards/6671" - } - ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(go_memstats_alloc_bytes_total{app=\"clickhouse-operator\",namespace=~\"$exported_namespace|kube-system\"}[1m])", - "hide": false, - "legendFormat": "{{ namespace }} GO malloc bytes / sec", - "refId": "A" - }, - { - "expr": "process_resident_memory_bytes{app=\"clickhouse-operator\",namespace=~\"$exported_namespace|kube-system\"}", - "legendFormat": "{{ namespace }} RSS Memory", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Monitoring Agent", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "description": "", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 74 - }, - "hiddenSeries": false, - "id": 55, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.15", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "exemplar": true, - "expr": "chi_clickhouse_metric_LoadAverage1{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", - "hide": false, - "interval": "", - "legendFormat": "{{hostname}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Load Average 1m", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:193", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:194", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" } ], "refresh": "1m", - "schemaVersion": 27, + "schemaVersion": 38, "style": "dark", - "tags": [ - "Altinity", - "clickhouse", - "operator" - ], + "tags": [], "templating": { "list": [ { @@ -4585,48 +5442,48 @@ "type": "datasource" }, { - "allValue": ".*", - "current": {}, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "K8S Namespace", - "multi": true, - "name": "exported_namespace", - "options": [], - "query": { - "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", - "refId": "clickhouse-operator-prometheus-exported_namespace-Variable-Query" + "current": { + "selected": false, + "text": "prometheus", + "value": "prometheus" }, - "refresh": 2, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, "regex": "", "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "datasource" }, { "allValue": ".*", - "current": {}, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=~\"$exported_namespace\"}, chi)", - "description": null, - "error": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, chi)", "hide": 0, "includeAll": true, - "label": "K8S Clickhouse Installation", + "label": "Cluster", "multi": true, "name": "chi", "options": [], "query": { - "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=~\"$exported_namespace\"}, chi)", - "refId": "clickhouse-operator-prometheus-chi-Variable-Query" + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, chi)", + "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", @@ -4640,11 +5497,20 @@ }, { "allValue": ".*", - "current": {}, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=~\"$exported_namespace\",chi=~\"$chi\"}, hostname)", - "description": null, - "error": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",chi=~\"$chi\"}, hostname)", "hide": 0, "includeAll": true, "label": "Server", @@ -4652,8 +5518,8 @@ "name": "hostname", "options": [], "query": { - "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=~\"$exported_namespace\",chi=~\"$chi\"}, hostname)", - "refId": "clickhouse-operator-prometheus-hostname-Variable-Query" + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",chi=~\"$chi\"}, hostname)", + "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", @@ -4698,5 +5564,6 @@ "timezone": "browser", "title": "Altinity ClickHouse Operator Dashboard", "uid": "clickhouse-operator", - "version": 20220908 + "version": 20092024, + "weekStart": "" } diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/ClickHouseKeeper_dashboard.json b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/ClickHouseKeeper_dashboard.json index 819cd519..566f6824 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/ClickHouseKeeper_dashboard.json +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/ClickHouseKeeper_dashboard.json @@ -9,31 +9,35 @@ "pluginName": "Prometheus" } ], + "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "7.1.1" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" + "version": "11.3.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" } ], "annotations": { "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -41,7 +45,10 @@ "type": "dashboard" }, { - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "enable": true, "expr": "ALERTS{app=~\"clickhouse-keeper.*\"}", "hide": false, @@ -55,882 +62,921 @@ ] }, "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, - "iteration": 1644828395145, "links": [], "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 0 }, - "hiddenSeries": false, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "11.3.0", "targets": [ { - "expr": "zk_avg_latency{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ClickHouseAsyncMetrics_KeeperAvgLatency{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", "interval": "", "legendFormat": "avg {{namespace}}.{{pod_name}}", "refId": "A" }, { - "expr": "zk_max_latency{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ClickHouseAsyncMetrics_KeeperMaxLatency{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", "interval": "", "legendFormat": "max {{namespace}}.{{pod_name}}", "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:118", - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:119", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 0 }, - "hiddenSeries": false, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "11.3.0", "targets": [ { - "expr": "zk_num_alive_connections{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ClickHouseMetrics_KeeperAliveConnections{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", "hide": false, "interval": "", "legendFormat": "{{namespace}}.{{pod_name}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Connections", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:118", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:119", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 0 }, - "hiddenSeries": false, "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "11.3.0", "targets": [ { - "expr": "irate(zk_packets_sent{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}[1m])", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(ClickHouseAsyncMetrics_KeeperPacketsSent{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}[1m])", "hide": false, "interval": "", "legendFormat": "OUT {{namespace}}.{{pod_name}}", "refId": "A" }, { - "expr": "-irate(zk_packets_received{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}[1m])", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "-irate(ClickHouseAsyncMetrics_KeeperPacketsReceived{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}[1m])", "interval": "", "legendFormat": "IN {{namespace}}.{{pod_name}}", "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Packets per second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:118", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:119", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 9 }, - "hiddenSeries": false, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "11.3.0", "targets": [ { - "expr": "zk_znode_count{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ClickHouseAsyncMetrics_KeeperZnodeCount{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", "interval": "", "legendFormat": "{{namespace}}.{{pod_name}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "ZNode count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:118", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:119", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 9 }, - "hiddenSeries": false, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "11.3.0", "targets": [ { - "expr": "zk_watch_count{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ClickHouseAsyncMetrics_KeeperWatchCount{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", "interval": "", "legendFormat": "{{namespace}}.{{pod_name}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Watch count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:118", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:119", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 9 }, - "hiddenSeries": false, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "11.3.0", "targets": [ { - "expr": "zk_ephemerals_count{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ClickHouseAsyncMetrics_KeeperEphemeralsCount{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", "interval": "", "legendFormat": "{{namespace}}.{{pod_name}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Ethermals nodes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:118", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:119", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Ephemeral Node count", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 18 }, - "hiddenSeries": false, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "11.3.0", "targets": [ { - "expr": "zk_approximate_data_size{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ClickHouseAsyncMetrics_KeeperApproximateDataSize{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", "interval": "", "legendFormat": "{{namespace}}.{{pod_name}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Memory data size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:118", - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:119", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 18 }, - "hiddenSeries": false, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:1855", - "alias": "/.*/", - "color": "#F2495C" + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "11.3.0", "targets": [ { - "expr": "irate(zk_outstanding_requests{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}[1m])", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(ClickHouseMetrics_KeeperOutstandingRequests{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}[1m])", "interval": "", "legendFormat": "{{namespace}}.{{pod_name}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Outstanding requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:118", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:119", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 18 }, - "hiddenSeries": false, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "11.3.0", "targets": [ { - "expr": "zk_open_file_descriptor_count{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ClickHouseAsyncMetrics_KeeperOpenFileDescriptorCount{namespace=~\"$namespace\", pod_name=~\"$pod_name\", container_name=\"clickhouse-keeper\"}", "interval": "", "legendFormat": "{{namespace}}.{{pod_name}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Open file descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:118", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:119", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" } ], - "refresh": false, - "schemaVersion": 26, - "style": "dark", + "refresh": "", + "schemaVersion": 40, "tags": [ "altinity", "clickhouse-keeper" @@ -958,46 +1004,44 @@ { "allValue": ".+", "current": {}, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "definition": "label_values(zk_ruok, namespace)", - "hide": 0, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(up{container_name=\"clickhouse-keeper\"},namespace)", "includeAll": true, - "label": null, "multi": true, "name": "namespace", "options": [], - "query": "label_values(zk_ruok, namespace)", + "query": { + "qryType": 1, + "query": "label_values(up{container_name=\"clickhouse-keeper\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, "refresh": 2, "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { "allValue": ".+", "current": {}, - "datasource": {"type":"prometheus","uid":"${ds_prometheus}"}, - "definition": "label_values(zk_ruok, pod_name)", - "hide": 0, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(up{container_name=\"clickhouse-keeper\", namespace=~\"$namespace\"},pod_name)", "includeAll": true, - "label": null, "multi": true, "name": "pod_name", "options": [], - "query": "label_values(zk_ruok, pod_name)", + "query": { + "qryType": 1, + "query": "label_values(up{container_name=\"clickhouse-keeper\", namespace=~\"$namespace\"},pod_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, "refresh": 2, "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" } ] }, @@ -1021,5 +1065,6 @@ "timezone": "", "title": "ClickHouseKeeper Dashboard", "uid": "clickhouse-keeper", - "version": 20220214 + "version": 2, + "weekStart": "" } diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/ClickHouse_Queries_dashboard.json b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/ClickHouse_Queries_dashboard.json index f348932a..ee27a17e 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/ClickHouse_Queries_dashboard.json +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/files/ClickHouse_Queries_dashboard.json @@ -147,8 +147,8 @@ "format": "time_series", "interval": "", "intervalFactor": 2, - "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", - "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 30\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / (t/1000 - lagInFrame(t/1000,1,0) OVER ()) ), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / (t/1000 - lagInFrame(t/1000,1,0) OVER ()) ), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 30\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -743,7 +743,7 @@ "interval": "", "intervalFactor": 2, "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\n", - "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n GROUP BY t ORDER BY t)", + "rawQuery": "SELECT t, c/(t/1000 - lagInFrame(t/1000,1,0) OVER ()) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n GROUP BY t ORDER BY t)", "refId": "A", "resultFormat": "time_series", "round": "0s", diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/_helpers.tpl b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/_helpers.tpl index ebac8ebf..f2d1aee2 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/_helpers.tpl +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/_helpers.tpl @@ -1,4 +1,15 @@ {{/* vim: set filetype=go-template: */}} +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "altinity-clickhouse-operator.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + {{/* Expand the name of the chart. */}} @@ -40,8 +51,8 @@ helm.sh/chart: {{ include "altinity-clickhouse-operator.chart" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} -{{- if .Values.podLabels }} -{{ toYaml .Values.podLabels }} +{{- if .Values.commonLabels }} +{{ toYaml .Values.commonLabels }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} @@ -54,6 +65,17 @@ app.kubernetes.io/name: {{ include "altinity-clickhouse-operator.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} +{{/* +Common annotations +*/}} +{{- define "altinity-clickhouse-operator.annotations" -}} +meta.helm.sh/release-name: {{ .Release.Name }} +meta.helm.sh/release-namespace: {{ .Release.Namespace }} +{{- if .Values.commonAnnotations }} +{{ toYaml .Values.commonAnnotations }} +{{- end -}} +{{- end -}} + {{/* Create the name of the service account to use */}} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-configmap.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-configmap.yaml new file mode 100644 index 00000000..ac87683c --- /dev/null +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-configmap.yaml @@ -0,0 +1,21 @@ +{{- if .Values.dashboards.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-dashboards + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} + labels: + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} + {{- if .Values.dashboards.additionalLabels }} + {{- toYaml .Values.dashboards.additionalLabels | nindent 4 }} + {{- end }} + annotations: + {{- include "altinity-clickhouse-operator.annotations" . | nindent 4 }} + {{- if .Values.dashboards.annotations }} + {{- toYaml .Values.dashboards.annotations | nindent 4 }} + {{- end }} +data: +{{- range $path, $_ := .Files.Glob "files/*.json" }} + {{ $path | trimPrefix "files/" }}: |- {{ $.Files.Get $path | nindent 4 -}} +{{ end }} +{{- end }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-secret.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-secret.yaml deleted file mode 100644 index 2ab5e98d..00000000 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/dashboards-secret.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if .Values.dashboards.enabled }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "altinity-clickhouse-operator.fullname" . }}-dashboards - namespace: {{ .Release.Namespace }} - labels: - {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} -{{- if .Values.dashboards.additionalLabels }} - {{- toYaml .Values.dashboards.additionalLabels | nindent 4 }} -{{- end }} -{{- with .Values.dashboards.annotations }} - annotations: - {{- toYaml . | nindent 4 }} -{{- end }} -type: Opaque -data: -{{- range $path, $_ := .Files.Glob "files/*.json" }} - {{ $path | trimPrefix "files/" }}: {{ $.Files.Get $path | b64enc -}} -{{ end }} -{{- end }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml index 03de65b3..387351a5 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.create -}} +{{- if (and .Values.rbac.create (not .Values.rbac.namespaceScoped)) -}} # Specifies either # ClusterRole # or @@ -12,7 +12,7 @@ metadata: name: {{ include "altinity-clickhouse-operator.fullname" . }} #namespace: kube-system labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} - namespace: {{ .Release.Namespace }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} rules: # # Core API group diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml index cf7b3e03..1f696a54 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.create -}} +{{- if (and .Values.rbac.create (not .Values.rbac.namespaceScoped)) -}} # Specifies either # ClusterRoleBinding between ClusterRole and ServiceAccount. # or @@ -11,7 +11,7 @@ metadata: name: {{ include "altinity-clickhouse-operator.fullname" . }} #namespace: kube-system labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} - namespace: {{ .Release.Namespace }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -19,5 +19,15 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "altinity-clickhouse-operator.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} + +# Template Parameters: +# +# NAMESPACE=kube-system +# COMMENT= +# ROLE_KIND=Role +# ROLE_NAME=clickhouse-operator +# ROLE_BINDING_KIND=RoleBinding +# ROLE_BINDING_NAME=clickhouse-operator +# {{- end }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml index 5e737e5d..a359e195 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml @@ -8,6 +8,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ printf "%s-confd-files" (include "altinity-clickhouse-operator.fullname" .) }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.confdFiles) | nindent 2 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml index 2a4615eb..6b1480b8 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml @@ -8,6 +8,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ printf "%s-configd-files" (include "altinity-clickhouse-operator.fullname" .) }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.configdFiles) | nindent 2 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml index 49a71235..bc6d21dd 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml @@ -8,6 +8,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ printf "%s-files" (include "altinity-clickhouse-operator.fullname" .) }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.files) | nindent 2 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml index ba41cd85..5302b1f2 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml @@ -8,6 +8,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ printf "%s-templatesd-files" (include "altinity-clickhouse-operator.fullname" .) }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.templatesdFiles) | nindent 2 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml index 42d44037..deca80c4 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml @@ -8,6 +8,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ printf "%s-usersd-files" (include "altinity-clickhouse-operator.fullname" .) }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.usersdFiles) | nindent 2 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-confd-files.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-confd-files.yaml new file mode 100644 index 00000000..03b8f18a --- /dev/null +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-confd-files.yaml @@ -0,0 +1,14 @@ +# Template Parameters: +# +# NAME=etc-keeper-operator-confd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-keeper-confd-files" (include "altinity-clickhouse-operator.fullname" .) }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} + labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} +data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.keeperConfdFiles) | nindent 2 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-configd-files.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-configd-files.yaml new file mode 100644 index 00000000..31b92289 --- /dev/null +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-configd-files.yaml @@ -0,0 +1,14 @@ +# Template Parameters: +# +# NAME=etc-keeper-operator-configd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-keeper-configd-files" (include "altinity-clickhouse-operator.fullname" .) }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} + labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} +data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.keeperConfigdFiles) | nindent 2 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-templatesd-files.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-templatesd-files.yaml new file mode 100644 index 00000000..f99569a8 --- /dev/null +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-templatesd-files.yaml @@ -0,0 +1,14 @@ +# Template Parameters: +# +# NAME=etc-keeper-operator-templatesd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-keeper-templatesd-files" (include "altinity-clickhouse-operator.fullname" .) }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} + labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} +data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.keeperTemplatesdFiles) | nindent 2 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-usersd-files.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-usersd-files.yaml new file mode 100644 index 00000000..9e3e8da3 --- /dev/null +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-keeper-operator-usersd-files.yaml @@ -0,0 +1,14 @@ +# Template Parameters: +# +# NAME=etc-keeper-operator-usersd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-keeper-usersd-files" (include "altinity-clickhouse-operator.fullname" .) }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} + labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} +data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.keeperUsersdFiles) | nindent 2 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml index 03b52fc5..08482ab1 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml @@ -2,9 +2,9 @@ # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.4 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.25.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.4 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.25.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -13,22 +13,27 @@ kind: Deployment apiVersion: apps/v1 metadata: name: {{ include "altinity-clickhouse-operator.fullname" . }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} spec: replicas: 1 selector: matchLabels: {{ include "altinity-clickhouse-operator.selectorLabels" . | nindent 6 }} template: metadata: - labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 8 }} + labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 8 }}{{ if .Values.podLabels }}{{ toYaml .Values.podLabels | nindent 8 }}{{ end }} annotations: - {{ toYaml .Values.podAnnotations | nindent 8 }} + {{ if .Values.podAnnotations }}{{ toYaml .Values.podAnnotations | nindent 8 }}{{ end }} checksum/files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-files.yaml") . | sha256sum }} checksum/confd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml") . | sha256sum }} checksum/configd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml") . | sha256sum }} checksum/templatesd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml") . | sha256sum }} checksum/usersd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml") . | sha256sum }} + checksum/keeper-confd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-keeper-operator-confd-files.yaml") . | sha256sum }} + checksum/keeper-configd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-keeper-operator-configd-files.yaml") . | sha256sum }} + checksum/keeper-templatesd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-keeper-operator-templatesd-files.yaml") . | sha256sum }} + checksum/keeper-usersd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-keeper-operator-usersd-files.yaml") . | sha256sum }} spec: serviceAccountName: {{ include "altinity-clickhouse-operator.serviceAccountName" . }} volumes: @@ -47,6 +52,18 @@ spec: - name: etc-clickhouse-operator-usersd-folder configMap: name: {{ include "altinity-clickhouse-operator.fullname" . }}-usersd-files + - name: etc-keeper-operator-confd-folder + configMap: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-keeper-confd-files + - name: etc-keeper-operator-configd-folder + configMap: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-keeper-configd-files + - name: etc-keeper-operator-templatesd-folder + configMap: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-keeper-templatesd-files + - name: etc-keeper-operator-usersd-folder + configMap: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-keeper-usersd-files containers: - name: {{ .Chart.Name }} image: {{ .Values.operator.image.repository }}:{{ include "altinity-clickhouse-operator.operator.tag" . }} @@ -55,13 +72,21 @@ spec: - name: etc-clickhouse-operator-folder mountPath: /etc/clickhouse-operator - name: etc-clickhouse-operator-confd-folder - mountPath: /etc/clickhouse-operator/conf.d + mountPath: /etc/clickhouse-operator/chi/conf.d - name: etc-clickhouse-operator-configd-folder - mountPath: /etc/clickhouse-operator/config.d + mountPath: /etc/clickhouse-operator/chi/config.d - name: etc-clickhouse-operator-templatesd-folder - mountPath: /etc/clickhouse-operator/templates.d + mountPath: /etc/clickhouse-operator/chi/templates.d - name: etc-clickhouse-operator-usersd-folder - mountPath: /etc/clickhouse-operator/users.d + mountPath: /etc/clickhouse-operator/chi/users.d + - name: etc-keeper-operator-confd-folder + mountPath: /etc/clickhouse-operator/chk/conf.d + - name: etc-keeper-operator-configd-folder + mountPath: /etc/clickhouse-operator/chk/keeper_config.d + - name: etc-keeper-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/chk/templates.d + - name: etc-keeper-operator-usersd-folder + mountPath: /etc/clickhouse-operator/chk/users.d env: # Pod-specific # spec.nodeName: ip-172-20-52-62.ec2.internal @@ -125,13 +150,21 @@ spec: - name: etc-clickhouse-operator-folder mountPath: /etc/clickhouse-operator - name: etc-clickhouse-operator-confd-folder - mountPath: /etc/clickhouse-operator/conf.d + mountPath: /etc/clickhouse-operator/chi/conf.d - name: etc-clickhouse-operator-configd-folder - mountPath: /etc/clickhouse-operator/config.d + mountPath: /etc/clickhouse-operator/chi/config.d - name: etc-clickhouse-operator-templatesd-folder - mountPath: /etc/clickhouse-operator/templates.d + mountPath: /etc/clickhouse-operator/chi/templates.d - name: etc-clickhouse-operator-usersd-folder - mountPath: /etc/clickhouse-operator/users.d + mountPath: /etc/clickhouse-operator/chi/users.d + - name: etc-keeper-operator-confd-folder + mountPath: /etc/clickhouse-operator/chk/conf.d + - name: etc-keeper-operator-configd-folder + mountPath: /etc/clickhouse-operator/chk/keeper_config.d + - name: etc-keeper-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/chk/templates.d + - name: etc-keeper-operator-usersd-folder + mountPath: /etc/clickhouse-operator/chk/users.d env: # Pod-specific # spec.nodeName: ip-172-20-52-62.ec2.internal @@ -193,3 +226,4 @@ spec: affinity: {{ toYaml .Values.affinity | nindent 8 }} tolerations: {{ toYaml .Values.tolerations | nindent 8 }} securityContext: {{ toYaml .Values.podSecurityContext | nindent 8 }} + topologySpreadConstraints: {{ toYaml .Values.topologySpreadConstraints | nindent 8 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Role-clickhouse-operator.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Role-clickhouse-operator.yaml new file mode 100644 index 00000000..2fb3015e --- /dev/null +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Role-clickhouse-operator.yaml @@ -0,0 +1,211 @@ +{{- if (and .Values.rbac.create .Values.rbac.namespaceScoped) -}} +# Specifies either +# ClusterRole +# or +# Role +# to be bound to ServiceAccount. +# ClusterRole is namespace-less and must have unique name +# Role is namespace-bound +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} + labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} +rules: + # + # Core API group + # + - apiGroups: + - "" + resources: + - configmaps + - services + - persistentvolumeclaims + - secrets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + - delete + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + # + # apps.* resources + # + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete + # The operator deployment personally, identified by name + - apiGroups: + - apps + resources: + - deployments + resourceNames: + - {{ include "altinity-clickhouse-operator.fullname" . }} + verbs: + - get + - patch + - update + - delete + # + # policy.* resources + # + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + # + # apiextensions + # + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + # clickhouse - related resources + - apiGroups: + - clickhouse.altinity.com + # + # The operators specific Custom Resources + # + + resources: + - clickhouseinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete + # clickhouse-keeper - related resources + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/finalizers + verbs: + - update + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get + - update + - patch + - create + - delete +{{- end }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/RoleBinding-clickhouse-operator.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/RoleBinding-clickhouse-operator.yaml new file mode 100644 index 00000000..50d4af15 --- /dev/null +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/RoleBinding-clickhouse-operator.yaml @@ -0,0 +1,23 @@ +{{- if (and .Values.rbac.create .Values.rbac.namespaceScoped) -}} +# Specifies either +# ClusterRoleBinding between ClusterRole and ServiceAccount. +# or +# RoleBinding between Role and ServiceAccount. +# ClusterRoleBinding is namespace-less and must have unique name +# RoleBinding is namespace-bound +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} + labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "altinity-clickhouse-operator.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "altinity-clickhouse-operator.serviceAccountName" . }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} +{{- end }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml index a5f68b9c..1c3664c1 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml @@ -3,7 +3,7 @@ # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.25.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -11,8 +11,9 @@ apiVersion: v1 kind: Secret metadata: name: {{ include "altinity-clickhouse-operator.fullname" . }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} type: Opaque data: username: {{ .Values.secret.username | b64enc }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml index 28a54e59..5ac9d4eb 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml @@ -12,8 +12,9 @@ kind: Service apiVersion: v1 metadata: name: {{ printf "%s-metrics" (include "altinity-clickhouse-operator.fullname" .) }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} spec: ports: - port: 8888 diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml index 3bc8d89a..bcc55c20 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml @@ -10,9 +10,9 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "altinity-clickhouse-operator.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} - annotations: {{ toYaml .Values.serviceAccount.annotations | nindent 4 }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }}{{ if .Values.serviceAccount.annotations }}{{ toYaml .Values.serviceAccount.annotations | nindent 4 }}{{ end }} # Template Parameters: # diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/servicemonitor.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/servicemonitor.yaml index f8dea8e6..da6d389b 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/servicemonitor.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/templates/servicemonitor.yaml @@ -3,16 +3,45 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: {{ printf "%s-clickhouse-metrics" (include "altinity-clickhouse-operator.fullname" .) }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} - {{- if .Values.serviceMonitor.additionalLabels }} + {{- if .Values.serviceMonitor.additionalLabels }} {{- toYaml .Values.serviceMonitor.additionalLabels | nindent 4 }} - {{- end }} + {{- end }} + annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} spec: endpoints: - port: clickhouse-metrics # 8888 + {{- with .Values.serviceMonitor.clickhouseMetrics.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.clickhouseMetrics.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.clickhouseMetrics.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.serviceMonitor.clickhouseMetrics.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} - port: operator-metrics # 9999 + {{- with .Values.serviceMonitor.operatorMetrics.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.operatorMetrics.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.operatorMetrics.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.serviceMonitor.operatorMetrics.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} selector: matchLabels: {{- include "altinity-clickhouse-operator.selectorLabels" . | nindent 6 }} diff --git a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/values.yaml b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/values.yaml index fbbf8972..9ffaab19 100644 --- a/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/values.yaml +++ b/packages/system/clickhouse-operator/charts/altinity-clickhouse-operator/values.yaml @@ -1,3 +1,8 @@ +namespaceOverride: "" +# commonLabels -- set of labels that will be applied to all the resources for the operator +commonLabels: {} +# commonAnnotations -- set of annotations that will be applied to all the resources for the operator +commonAnnotations: {} operator: image: # operator.image.repository -- image repository @@ -7,7 +12,7 @@ operator: # operator.image.pullPolicy -- image pull policy pullPolicy: IfNotPresent containerSecurityContext: {} - # operator.resources -- custom resource configuration, look `kubectl explain pod.spec.containers.resources` for details + # operator.resources -- custom resource configuration, check `kubectl explain pod.spec.containers.resources` for details resources: {} # limits: # cpu: 100m @@ -17,7 +22,7 @@ operator: # memory: 128Mi # operator.env -- additional environment variables for the clickhouse-operator container in deployment - # possible format value [{"name": "SAMPLE", "value": "text"}] + # possible format value `[{"name": "SAMPLE", "value": "text"}]` env: [] metrics: enabled: true @@ -39,15 +44,16 @@ metrics: # memory: 128Mi # metrics.env -- additional environment variables for the deployment of metrics-exporter containers - # possible format value [{"name": "SAMPLE", "value": "text"}] + # possible format value `[{"name": "SAMPLE", "value": "text"}]` env: [] # imagePullSecrets -- image pull secret for private images in clickhouse-operator pod -# possible value format [{"name":"your-secret-name"}] -# look `kubectl explain pod.spec.imagePullSecrets` for details +# possible value format `[{"name":"your-secret-name"}]`, +# check `kubectl explain pod.spec.imagePullSecrets` for details imagePullSecrets: [] # podLabels -- labels to add to the clickhouse-operator pod podLabels: {} -# podAnnotations -- annotations to add to the clickhouse-operator pod, look `kubectl explain pod.spec.annotations` for details +# podAnnotations -- annotations to add to the clickhouse-operator pod, check `kubectl explain pod.spec.annotations` for details +# @default -- check the `values.yaml` file podAnnotations: prometheus.io/port: '8888' prometheus.io/scrape: 'true' @@ -65,8 +71,10 @@ serviceAccount: # serviceAccount.name -- the name of the service account to use; if not set and create is true, a name is generated using the fullname template name: rbac: - # rbac.create -- specifies whether cluster roles and cluster role bindings should be created + # rbac.create -- specifies whether rbac resources should be created create: true + # rbac.namespaceScoped -- specifies whether to create roles and rolebindings at the cluster level or namespace level + namespaceScoped: false secret: # secret.create -- create a secret with operator credentials create: true @@ -74,21 +82,42 @@ secret: username: clickhouse_operator # secret.password -- operator credentials password password: clickhouse_operator_password -# nodeSelector -- node for scheduler pod assignment, look `kubectl explain pod.spec.nodeSelector` for details +# nodeSelector -- node for scheduler pod assignment, check `kubectl explain pod.spec.nodeSelector` for details nodeSelector: {} -# tolerations -- tolerations for scheduler pod assignment, look `kubectl explain pod.spec.tolerations` for details +# tolerations -- tolerations for scheduler pod assignment, check `kubectl explain pod.spec.tolerations` for details tolerations: [] -# affinity -- affinity for scheduler pod assignment, look `kubectl explain pod.spec.affinity` for details +# affinity -- affinity for scheduler pod assignment, check `kubectl explain pod.spec.affinity` for details affinity: {} -# podSecurityContext - operator deployment SecurityContext, look `kubectl explain pod.spec.securityContext` for details +# podSecurityContext - operator deployment SecurityContext, check `kubectl explain pod.spec.securityContext` for details podSecurityContext: {} +# topologySpreadConstraints - topologySpreadConstraints affinity for scheduler pod assignment, check `kubectl explain pod.spec.topologySpreadConstraints` for details +topologySpreadConstraints: [] serviceMonitor: - # serviceMonitor.enabled -- ServiceMonitor Custom resource is created for a (prometheus-operator)[https://github.com/prometheus-operator/prometheus-operator] + # serviceMonitor.enabled -- ServiceMonitor Custom resource is created for a [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) + # In serviceMonitor will be created two endpoints clickhouse-metrics on port 8888 and operator-metrics # 9999. Ypu can specify interval, scrapeTimeout, relabelings, metricRelabelings for each endpoint below enabled: false # serviceMonitor.additionalLabels -- additional labels for service monitor additionalLabels: {} -# configs -- clickhouse-operator configs -# @default -- check the values.yaml file for the config content, auto-generated from latest operator release + clickhouseMetrics: + # serviceMonitor.interval for clickhouse-metrics endpoint -- + interval: 30s + # serviceMonitor.scrapeTimeout for clickhouse-metrics endpoint -- Prometheus ServiceMonitor scrapeTimeout. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used. + scrapeTimeout: "" + # serviceMonitor.relabelings for clickhouse-metrics endpoint -- Prometheus [RelabelConfigs] to apply to samples before scraping + relabelings: [] + # serviceMonitor.metricRelabelings for clickhouse-metrics endpoint -- Prometheus [MetricRelabelConfigs] to apply to samples before ingestio + metricRelabelings: [] + operatorMetrics: + # serviceMonitor.interval for operator-metrics endpoint -- + interval: 30s + # serviceMonitor.scrapeTimeout for operator-metrics endpoint -- Prometheus ServiceMonitor scrapeTimeout. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used. + scrapeTimeout: "" + # serviceMonitor.relabelings for operator-metrics endpoint -- Prometheus [RelabelConfigs] to apply to samples before scraping + relabelings: [] + # serviceMonitor.metricRelabelings for operator-metrics endpoint-- Prometheus [MetricRelabelConfigs] to apply to samples before ingestio + metricRelabelings: [] +# configs -- clickhouse operator configs +# @default -- check the `values.yaml` file for the config content (auto-generated from latest operator release) configs: confdFiles: null configdFiles: @@ -212,12 +241,12 @@ configs: # In case path is relative - it is relative to the folder where configuration file you are reading right now is located. path: # Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. - common: config.d + common: chi/config.d # Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. - host: conf.d + host: chi/conf.d # Path to the folder where ClickHouse configuration files with users' settings are located. # Files are common for all instances within a CHI. - user: users.d + user: chi/users.d ################################################ ## ## Configuration users section @@ -287,10 +316,13 @@ configs: - settings/macros/*: "no" - settings/remote_servers/*: "no" - settings/user_directories/*: "no" + # these settings should not lead to pod restarts + - settings/display_secrets_in_show_and_select: "no" - zookeeper/*: "yes" - files/*.xml: "yes" - files/config.d/*.xml: "yes" - files/config.d/*dict*.xml: "no" + - files/config.d/*no_restart*: "no" # exceptions in default profile - profiles/default/background_*_pool_size: "yes" - profiles/default/max_*_for_server: "yes" @@ -312,7 +344,6 @@ configs: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" @@ -339,6 +370,56 @@ configs: connect: 1 # Timout to perform SQL query from the operator to ClickHouse instances. In seconds. query: 4 + ################################################ + ## + ## Addons specifies additional configuration sections + ## Should it be called something like "templates"? + ## + ################################################ + addons: + rules: + - version: "*" + spec: + configuration: + users: + profiles: + quotas: + settings: + files: + - version: ">= 23.3" + spec: + configuration: + ### + ### users.d is global while description depends on CH version which may vary on per-host basis + ### In case of global-ness this may be better to implement via auto-templates + ### + ### As a solution, this may be applied on the whole cluster based on any of its hosts + ### + ### What to do when host is just created? CH version is not known prior to CH started and user config is required before CH started. + ### We do not have any info about the cluster on initial creation + ### + users: + "{clickhouseOperatorUser}/access_management": 1 + "{clickhouseOperatorUser}/named_collection_control": 1 + "{clickhouseOperatorUser}/show_named_collections": 1 + "{clickhouseOperatorUser}/show_named_collections_secrets": 1 + profiles: + quotas: + settings: + files: + - version: ">= 23.5" + spec: + configuration: + users: + profiles: + clickhouse_operator/format_display_secrets_in_show_and_select: 1 + quotas: + settings: + ## + ## this may be added on per-host basis into host's conf.d folder + ## + display_secrets_in_show_and_select: 1 + files: ################################################# ## ## Metrics collection @@ -352,6 +433,25 @@ configs: # Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. # All collected metrics are returned. collect: 9 + keeper: + configuration: + ################################################ + ## + ## Configuration files section + ## + ################################################ + file: + # Each 'path' can be either absolute or relative. + # In case path is absolute - it is used as is + # In case path is relative - it is relative to the folder where configuration file you are reading right now is located. + path: + # Path to the folder where Keeper configuration files common for all instances within a CHK are located. + common: chk/keeper_config.d + # Path to the folder where Keeper configuration files unique for each instance (host) within a CHK are located. + host: chk/conf.d + # Path to the folder where Keeper configuration files with users' settings are located. + # Files are common for all instances within a CHI. + user: chk/users.d ################################################ ## ## Template(s) management section @@ -367,7 +467,17 @@ configs: # Path to the folder where ClickHouseInstallation templates .yaml manifests are located. # Templates are added to the list of all templates and used when CHI is reconciled. # Templates are applied in sorted alpha-numeric order. - path: templates.d + path: chi/templates.d + chk: + # CHK template updates handling policy + # Possible policy values: + # - ReadOnStart. Accept CHIT updates on the operators start only. + # - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI + policy: ApplyOnNextReconcile + # Path to the folder where ClickHouseInstallation templates .yaml manifests are located. + # Templates are added to the list of all templates and used when CHI is reconciled. + # Templates are applied in sorted alpha-numeric order. + path: chk/templates.d ################################################ ## ## Reconcile section @@ -386,9 +496,9 @@ configs: # 3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward. # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage - # Max number of concurrent shard reconciles within one CHI in progress + # Max number of concurrent shard reconciles within one cluster in progress reconcileShardsThreadsNumber: 5 - # Max percentage of concurrent shard reconciles within one CHI in progress + # Max percentage of concurrent shard reconciles within one cluster in progress reconcileShardsMaxConcurrencyPercent: 50 # Reconcile StatefulSet scenario statefulSet: @@ -429,6 +539,10 @@ configs: exclude: true queries: true include: false + replicas: + all: no + new: yes + delay: 10 ################################################ ## ## Annotations management section @@ -473,6 +587,25 @@ configs: appendScope: "no" ################################################ ## + ## Metrics management section + ## + ################################################ + metrics: + labels: + exclude: [] + ################################################ + ## + ## Status management section + ## + ################################################ + status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ + ## ## StatefulSet management section ## ################################################ @@ -631,20 +764,87 @@ configs: -# additionalResources -- list of additional resources to create (are processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator, look `kubectl explain chi` for details + keeperConfdFiles: null + keeperConfigdFiles: + 01-keeper-01-default-config.xml: | + + + + + + + + + + 10000 + 10000 + information + 100000 + + true + /var/lib/clickhouse-keeper/coordination/logs + /var/lib/clickhouse-keeper/coordination/snapshots + /var/lib/clickhouse-keeper + 2181 + + :: + 0.0.0.0 + 1 + + 1 + information + + 4096 + + 01-keeper-02-readiness.xml: | + + + + + + + + + + 9182 + + /ready + + + + + 01-keeper-03-enable-reconfig.xml: |- + + + + + + + + + false + + + keeperTemplatesdFiles: + readme: |- + Templates in this folder are packaged with an operator and available via 'useTemplate' + keeperUsersdFiles: null +# additionalResources -- list of additional resources to create (processed via `tpl` function), +# useful for create ClickHouse clusters together with clickhouse-operator. +# check `kubectl explain chi` for details additionalResources: [] # - | # apiVersion: v1 # kind: ConfigMap # metadata: # name: {{ include "altinity-clickhouse-operator.fullname" . }}-cm -# namespace: {{ .Release.Namespace }} +# namespace: {{ include "altinity-clickhouse-operator.namespace" . }} # - | # apiVersion: v1 # kind: Secret # metadata: # name: {{ include "altinity-clickhouse-operator.fullname" . }}-s -# namespace: {{ .Release.Namespace }} +# namespace: {{ include "altinity-clickhouse-operator.namespace" . }} # stringData: # mykey: my-value # - | @@ -652,15 +852,16 @@ additionalResources: [] # kind: ClickHouseInstallation # metadata: # name: {{ include "altinity-clickhouse-operator.fullname" . }}-chi -# namespace: {{ .Release.Namespace }} +# namespace: {{ include "altinity-clickhouse-operator.namespace" . }} # spec: # configuration: # clusters: # - name: default # layout: # shardsCount: 1 + dashboards: - # dashboards.enabled -- provision grafana dashboards as secrets (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-6.33.1/charts/grafana/values.yaml#L679 ) + # dashboards.enabled -- provision grafana dashboards as configMaps (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-8.3.4/charts/grafana/values.yaml#L778 ) enabled: false # dashboards.additionalLabels -- labels to add to a secret with dashboards additionalLabels: From 299d006d203f951c5cd507c90aa70fdc1041ffc3 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 4 Aug 2025 23:31:08 +0300 Subject: [PATCH 24/30] [test] fix vm tests Signed-off-by: IvanHunters --- hack/e2e-apps/virtualmachine.bats | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hack/e2e-apps/virtualmachine.bats b/hack/e2e-apps/virtualmachine.bats index a0bbe8f4..33e87b53 100644 --- a/hack/e2e-apps/virtualmachine.bats +++ b/hack/e2e-apps/virtualmachine.bats @@ -20,9 +20,7 @@ spec: storage: 5Gi storageClass: replicated gpus: [] - resources: - cpu: "" - memory: "" + resources: {} sshKeys: - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF test@test From cd80a7344600d491399c45cf3cb28b8514c4fdb3 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 5 Aug 2025 12:54:47 +0200 Subject: [PATCH 25/30] [dashboard] fix diff editor Signed-off-by: Andrei Kvapil --- packages/system/dashboard/images/dashboard/Dockerfile | 2 +- packages/system/dashboard/images/kubeapps-apis/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/system/dashboard/images/dashboard/Dockerfile b/packages/system/dashboard/images/dashboard/Dockerfile index 633d9e7f..48656a2c 100644 --- a/packages/system/dashboard/images/dashboard/Dockerfile +++ b/packages/system/dashboard/images/dashboard/Dockerfile @@ -1,7 +1,7 @@ FROM bitnami/node:20.15.1 AS build WORKDIR /app -ARG COMMIT_REF=cdf9095f50c74505870de337725d2a9d0bd20947 +ARG COMMIT_REF=4926bc68fabb0914afab574006643c85a597b371 RUN wget -O- https://github.com/cozystack/kubeapps/archive/${COMMIT_REF}.tar.gz | tar xzf - --strip-components=2 kubeapps-${COMMIT_REF}/dashboard RUN yarn install --frozen-lockfile diff --git a/packages/system/dashboard/images/kubeapps-apis/Dockerfile b/packages/system/dashboard/images/kubeapps-apis/Dockerfile index 619f9386..a525a170 100644 --- a/packages/system/dashboard/images/kubeapps-apis/Dockerfile +++ b/packages/system/dashboard/images/kubeapps-apis/Dockerfile @@ -4,7 +4,7 @@ # syntax = docker/dockerfile:1 FROM alpine AS source -ARG COMMIT_REF=cdf9095f50c74505870de337725d2a9d0bd20947 +ARG COMMIT_REF=4926bc68fabb0914afab574006643c85a597b371 RUN apk add --no-cache patch WORKDIR /source RUN wget -O- https://github.com/cozystack/kubeapps/archive/${COMMIT_REF}.tar.gz | tar xzf - --strip-components=1 From c4a2bef4c91181208a6820d879c0b2a1bc26fafc Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 4 Aug 2025 23:31:08 +0300 Subject: [PATCH 26/30] [test] fix vm tests Signed-off-by: IvanHunters (cherry picked from commit 299d006d203f951c5cd507c90aa70fdc1041ffc3) --- hack/e2e-apps/virtualmachine.bats | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hack/e2e-apps/virtualmachine.bats b/hack/e2e-apps/virtualmachine.bats index a0bbe8f4..33e87b53 100644 --- a/hack/e2e-apps/virtualmachine.bats +++ b/hack/e2e-apps/virtualmachine.bats @@ -20,9 +20,7 @@ spec: storage: 5Gi storageClass: replicated gpus: [] - resources: - cpu: "" - memory: "" + resources: {} sshKeys: - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF test@test From 754d5a976db02df5d1482237eef280061003d10b Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Fri, 18 Jul 2025 23:39:12 +0200 Subject: [PATCH 27/30] [apps] Introduce new OpenAPI schema generator Use https://github.com/cozystack/cozyvalues-gen for three apps: - apps/postgres - apps/virtual-machine - extra/monitoring Changes: - Add type and enum definitions to values.yaml. - Update READMEs with new information. - Update values.schema.json with definitions for children objects, allowing precise UI customization. Add regexp for specific types such as resources: CPU like `500m` and RAM like `4GiB`. - Remove direct injections with `yq` from Makefiles where they're not needed anymore. Co-authored-by: Nick Volynkin Signed-off-by: Andrei Kvapil Signed-off-by: Nick Volynkin --- .github/workflows/pre-commit.yml | 1 + packages/apps/postgres/Makefile | 4 +- packages/apps/postgres/README.md | 75 ++- packages/apps/postgres/values.schema.json | 379 ++++++++----- packages/apps/postgres/values.yaml | 67 ++- packages/apps/virtual-machine/Makefile | 5 +- packages/apps/virtual-machine/README.md | 40 +- .../apps/virtual-machine/values.schema.json | 106 ++-- packages/apps/virtual-machine/values.yaml | 57 +- packages/extra/monitoring/Makefile | 6 +- packages/extra/monitoring/README.md | 104 +++- packages/extra/monitoring/values.schema.json | 533 ++++++++++++++---- packages/extra/monitoring/values.yaml | 69 ++- 13 files changed, 1026 insertions(+), 420 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a06a171f..e00fe0d7 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -29,6 +29,7 @@ jobs: - name: Install generate run: | curl -sSL https://github.com/cozystack/readme-generator-for-helm/releases/download/v1.0.0/readme-generator-for-helm-linux-amd64.tar.gz | tar -xzvf- -C /usr/local/bin/ readme-generator-for-helm + curl -sSL https://github.com/cozystack/cozyvalues-gen/releases/download/v0.7.0/cozyvalues-gen-linux-amd64.tar.gz | tar -xzvf- -C /usr/local/bin/ cozyvalues-gen - name: Run pre-commit hooks run: | diff --git a/packages/apps/postgres/Makefile b/packages/apps/postgres/Makefile index 4a92cb71..7017c673 100644 --- a/packages/apps/postgres/Makefile +++ b/packages/apps/postgres/Makefile @@ -1,6 +1,4 @@ include ../../../scripts/package.mk -PRESET_ENUM := ["nano","micro","small","medium","large","xlarge","2xlarge"] generate: - readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md - yq -i -o json --indent 4 '.properties.resourcesPreset.enum = $(PRESET_ENUM)' values.schema.json + cozyvalues-gen -v values.yaml -s values.schema.json -r README.md diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index e0a17ec6..c9cda3c3 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -66,44 +66,61 @@ See: ### Common parameters -| Name | Description | Value | -| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ------- | -| `replicas` | Number of Postgres replicas | `2` | -| `resources` | Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied. | `{}` | -| `resourcesPreset` | Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge. | `micro` | -| `size` | Persistent Volume size | `10Gi` | -| `storageClass` | StorageClass used to store the data | `""` | -| `external` | Enable external access from outside the cluster | `false` | +| Name | Description | Type | Value | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------- | +| `replicas` | Number of Postgres replicas | `int` | `2` | +| `resources` | Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied. | `*object` | `{}` | +| `resources.cpu` | CPU | `*string` | `null` | +| `resources.memory` | Memory | `*string` | `null` | +| `resourcesPreset` | Default sizing preset used when `resources` is omitted. Allowed values: `nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`. | `string` | `{}` | +| `size` | Persistent Volume Claim size, available for application data | `string` | `10Gi` | +| `storageClass` | StorageClass used to store the data | `string` | `""` | +| `external` | Enable external access from outside the cluster | `bool` | `false` | + ### Application-specific parameters -| Name | Description | Value | -| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ----- | -| `postgresql.parameters.max_connections` | Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections | `100` | -| `quorum.minSyncReplicas` | Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed. | `0` | -| `quorum.maxSyncReplicas` | Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances). | `0` | -| `users` | Users configuration | `{}` | -| `databases` | Databases configuration | `{}` | +| Name | Description | Type | Value | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ------------------- | ------- | +| `postgresql` | PostgreSQL server configuration | `object` | `{}` | +| `postgresql.parameters` | PostgreSQL server parameters | `object` | `{}` | +| `postgresql.parameters.max_connections` | Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections | `int` | `100` | +| `quorum` | Quorum configuration for synchronous replication | `object` | `{}` | +| `quorum.minSyncReplicas` | Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed. | `int` | `0` | +| `quorum.maxSyncReplicas` | Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances). | `int` | `0` | +| `users` | Users configuration | `map[string]object` | `{...}` | +| `users[name].password` | Password for the user | `*string` | `null` | +| `users[name].replication` | Whether the user has replication privileges | `*bool` | `null` | +| `databases` | Databases configuration | `map[string]object` | `{...}` | +| `databases[name].roles` | Roles for the database | `*object` | `null` | +| `databases[name].roles.admin` | List of users with admin privileges | `[]string` | `[]` | +| `databases[name].roles.readonly` | List of users with read-only privileges | `[]string` | `[]` | +| `databases[name].extensions` | Extensions enabled for the database | `[]string` | `[]` | + ### Backup parameters -| Name | Description | Value | -| ------------------------ | ---------------------------------------------------------- | ----------------------------------- | -| `backup.enabled` | Enable regular backups | `false` | -| `backup.schedule` | Cron schedule for automated backups | `0 2 * * * *` | -| `backup.retentionPolicy` | Retention policy | `30d` | -| `backup.destinationPath` | Path to store the backup (i.e. s3://bucket/path/to/folder) | `s3://bucket/path/to/folder/` | -| `backup.endpointURL` | S3 Endpoint used to upload data to the cloud | `http://minio-gateway-service:9000` | -| `backup.s3AccessKey` | Access key for S3, used for authentication | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu` | -| `backup.s3SecretKey` | Secret key for S3, used for authentication | `ju3eum4dekeich9ahM1te8waeGai0oog` | +| Name | Description | Type | Value | +| ------------------------ | ---------------------------------------------------------- | -------- | ----------------------------------- | +| `backup` | Backup configuration | `object` | `{}` | +| `backup.enabled` | Enable regular backups | `bool` | `false` | +| `backup.schedule` | Cron schedule for automated backups | `string` | `0 2 * * * *` | +| `backup.retentionPolicy` | Retention policy | `string` | `30d` | +| `backup.destinationPath` | Path to store the backup (i.e. s3://bucket/path/to/folder) | `string` | `s3://bucket/path/to/folder/` | +| `backup.endpointURL` | S3 Endpoint used to upload data to the cloud | `string` | `http://minio-gateway-service:9000` | +| `backup.s3AccessKey` | Access key for S3, used for authentication | `string` | `` | +| `backup.s3SecretKey` | Secret key for S3, used for authentication | `string` | `` | + ### Bootstrap (recovery) parameters -| Name | Description | Value | -| ------------------------ | -------------------------------------------------------------------------------------------------------------------- | ------- | -| `bootstrap.enabled` | Restore database cluster from a backup | `false` | -| `bootstrap.recoveryTime` | Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest | `""` | -| `bootstrap.oldName` | Name of database cluster before deleting | `""` | +| Name | Description | Type | Value | +| ------------------------ | -------------------------------------------------------------------------------------------------------------------- | --------- | ------- | +| `bootstrap` | Bootstrap configuration | `object` | `{}` | +| `bootstrap.enabled` | Restore database cluster from a backup | `bool` | `false` | +| `bootstrap.recoveryTime` | Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest | `*string` | `""` | +| `bootstrap.oldName` | Name of database cluster before deleting | `string` | `""` | + ## Parameter examples and reference diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 6964fde1..9ba39ebc 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -1,140 +1,257 @@ { - "properties": { - "backup": { - "properties": { - "destinationPath": { - "default": "s3://bucket/path/to/folder/", - "description": "Path to store the backup (i.e. s3://bucket/path/to/folder)", - "type": "string" - }, - "enabled": { - "default": false, - "description": "Enable regular backups", - "type": "boolean" - }, - "endpointURL": { - "default": "http://minio-gateway-service:9000", - "description": "S3 Endpoint used to upload data to the cloud", - "type": "string" - }, - "retentionPolicy": { - "default": "30d", - "description": "Retention policy", - "type": "string" - }, - "s3AccessKey": { - "default": "oobaiRus9pah8PhohL1ThaeTa4UVa7gu", - "description": "Access key for S3, used for authentication", - "type": "string" - }, - "s3SecretKey": { - "default": "ju3eum4dekeich9ahM1te8waeGai0oog", - "description": "Secret key for S3, used for authentication", - "type": "string" - }, - "schedule": { - "default": "0 2 * * * *", - "description": "Cron schedule for automated backups", - "type": "string" - } - }, - "type": "object" + "title": "Chart Values", + "type": "object", + "properties": { + "backup": { + "description": "Backup configuration", + "type": "object", + "default": { + "destinationPath": "s3://bucket/path/to/folder/", + "enabled": false, + "endpointURL": "http://minio-gateway-service:9000", + "retentionPolicy": "30d", + "s3AccessKey": "\u003caccess key\u003e", + "s3SecretKey": "\u003csecret key\u003e", + "schedule": "0 2 * * * *" + }, + "required": [ + "destinationPath", + "enabled", + "endpointURL", + "retentionPolicy", + "s3AccessKey", + "s3SecretKey", + "schedule" + ], + "properties": { + "destinationPath": { + "description": "Path to store the backup (i.e. s3://bucket/path/to/folder)", + "type": "string", + "default": "s3://bucket/path/to/folder/" }, - "bootstrap": { - "properties": { - "enabled": { - "default": false, - "description": "Restore database cluster from a backup", - "type": "boolean" - }, - "oldName": { - "default": "", - "description": "Name of database cluster before deleting", - "type": "string" - }, - "recoveryTime": { - "default": "", - "description": "Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest", - "type": "string" - } - }, - "type": "object" + "enabled": { + "description": "Enable regular backups", + "type": "boolean", + "default": false }, - "databases": { - "default": {}, - "description": "Databases configuration", - "type": "object" + "endpointURL": { + "description": "S3 Endpoint used to upload data to the cloud", + "type": "string", + "default": "http://minio-gateway-service:9000" }, - "external": { - "default": false, - "description": "Enable external access from outside the cluster", - "type": "boolean" + "retentionPolicy": { + "description": "Retention policy", + "type": "string", + "default": "30d" }, - "postgresql": { - "properties": { - "parameters": { - "properties": { - "max_connections": { - "default": 100, - "description": "Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections", - "type": "number" - } - }, - "type": "object" - } - }, - "type": "object" + "s3AccessKey": { + "description": "Access key for S3, used for authentication", + "type": "string", + "default": "\u003caccess key\u003e" }, - "quorum": { - "properties": { - "maxSyncReplicas": { - "default": 0, - "description": "Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).", - "type": "number" - }, - "minSyncReplicas": { - "default": 0, - "description": "Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.", - "type": "number" - } - }, - "type": "object" + "s3SecretKey": { + "description": "Secret key for S3, used for authentication", + "type": "string", + "default": "\u003csecret key\u003e" }, - "replicas": { - "default": 2, - "description": "Number of Postgres replicas", - "type": "number" - }, - "resources": { - "default": {}, - "description": "Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied.", - "type": "object" - }, - "resourcesPreset": { - "default": "micro", - "description": "Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge.", - "type": "string", - "enum": [ - "nano", - "micro", - "small", - "medium", - "large", - "xlarge", - "2xlarge" - ] - }, - "size": { - "default": "10Gi", - "description": "Persistent Volume size", - "type": "string" - }, - "storageClass": { - "default": "", - "description": "StorageClass used to store the data", - "type": "string" + "schedule": { + "description": "Cron schedule for automated backups", + "type": "string", + "default": "0 2 * * * *" } + } }, - "title": "Chart Values", - "type": "object" -} + "bootstrap": { + "description": "Bootstrap configuration", + "type": "object", + "default": { + "enabled": false, + "oldName": "", + "recoveryTime": "" + }, + "required": [ + "enabled", + "oldName" + ], + "properties": { + "enabled": { + "description": "Restore database cluster from a backup", + "type": "boolean", + "default": false + }, + "oldName": { + "description": "Name of database cluster before deleting", + "type": "string" + }, + "recoveryTime": { + "description": "Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest", + "type": "string" + } + } + }, + "databases": { + "description": "Databases configuration", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "object", + "properties": { + "extensions": { + "description": "Extensions enabled for the database", + "type": "array", + "items": { + "type": "string" + } + }, + "roles": { + "description": "Roles for the database", + "type": "object", + "properties": { + "admin": { + "description": "List of users with admin privileges", + "type": "array", + "items": { + "type": "string" + } + }, + "readonly": { + "description": "List of users with read-only privileges", + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "external": { + "description": "Enable external access from outside the cluster", + "type": "boolean", + "default": false + }, + "postgresql": { + "description": "PostgreSQL server configuration", + "type": "object", + "default": { + "parameters": { + "max_connections": 100 + } + }, + "required": [ + "parameters" + ], + "properties": { + "parameters": { + "description": "PostgreSQL server parameters", + "type": "object", + "default": { + "max_connections": 100 + }, + "required": [ + "max_connections" + ], + "properties": { + "max_connections": { + "description": "Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections", + "type": "integer", + "default": 100 + } + } + } + } + }, + "quorum": { + "description": "Quorum configuration for synchronous replication", + "type": "object", + "default": { + "maxSyncReplicas": 0, + "minSyncReplicas": 0 + }, + "required": [ + "maxSyncReplicas", + "minSyncReplicas" + ], + "properties": { + "maxSyncReplicas": { + "description": "Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).", + "type": "integer", + "default": 0 + }, + "minSyncReplicas": { + "description": "Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.", + "type": "integer", + "default": 0 + } + } + }, + "replicas": { + "description": "Number of Postgres replicas", + "type": "integer", + "default": 2 + }, + "resources": { + "description": "Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied.", + "type": "object", + "default": {}, + "properties": { + "cpu": { + "description": "CPU", + "type": "string", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory", + "type": "string", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + }, + "resourcesPreset": { + "description": "Default sizing preset used when `resources` is omitted. Allowed values: `nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`.", + "type": "string", + "default": "micro", + "enum": [ + "nano", + "micro", + "small", + "medium", + "large", + "xlarge", + "2xlarge" + ] + }, + "size": { + "description": "Persistent Volume Claim size, available for application data", + "type": "string", + "default": "10Gi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "storageClass": { + "description": "StorageClass used to store the data", + "type": "string" + }, + "users": { + "description": "Users configuration", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "object", + "properties": { + "password": { + "description": "Password for the user", + "type": "string" + }, + "replication": { + "description": "Whether the user has replication privileges", + "type": "boolean" + } + } + } + } + } +} \ No newline at end of file diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index 25aa091e..bd283477 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -1,36 +1,44 @@ ## @section Common parameters ## -## @param replicas Number of Postgres replicas +## @param replicas {int} Number of Postgres replicas replicas: 2 -## @param resources Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied. +## @param resources {*resources} Explicit CPU and memory configuration for each PostgreSQL replica. When left empty, the preset defined in `resourcesPreset` is applied. +## @field resources.cpu {*quantity} CPU +## @field resources.memory {*quantity} Memory resources: {} # resources: # cpu: 4000m # memory: 4Gi -## @param resourcesPreset Default sizing preset used when `resources` is omitted. Allowed values: nano, micro, small, medium, large, xlarge, 2xlarge. +## @param resourcesPreset {string enum:"nano,micro,small,medium,large,xlarge,2xlarge"} Default sizing preset used when `resources` is omitted. Allowed values: `nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`. resourcesPreset: "micro" -## @param size Persistent Volume size +## @param size {quantity} Persistent Volume Claim size, available for application data size: 10Gi -## @param storageClass StorageClass used to store the data +## @param storageClass {string} StorageClass used to store the data storageClass: "" -## @param external Enable external access from outside the cluster +## @param external {bool} Enable external access from outside the cluster external: false - ## @section Application-specific parameters +## @param postgresql {postgresql} PostgreSQL server configuration +## @field postgresql.parameters {postgresqlParameters} PostgreSQL server parameters +## @field postgresqlParameters.max_connections {int} Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections ## -## @param postgresql.parameters.max_connections Determines the maximum number of concurrent connections to the database server. The default is typically 100 connections postgresql: parameters: max_connections: 100 -## @param quorum.minSyncReplicas Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed. -## @param quorum.maxSyncReplicas Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances). +## Configuration for the quorum-based synchronous replication +## @param quorum {quorum} Quorum configuration for synchronous replication +## @field quorum.minSyncReplicas {int} Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed. +## @field quorum.maxSyncReplicas {int} Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances). quorum: minSyncReplicas: 0 maxSyncReplicas: 0 -## @param users [object] Users configuration +## @param users {map[string]user} Users configuration +## @field user.password {*string} Password for the user +## @field user.replication {*bool} Whether the user has replication privileges +## ## Example: ## users: ## user1: @@ -44,7 +52,12 @@ quorum: ## users: {} -## @param databases Databases configuration +## @param databases {map[string]database} Databases configuration +## @field database.roles {*databaseRoles} Roles for the database +## @field databaseRoles.admin {[]string} List of users with admin privileges +## @field databaseRoles.readonly {[]string} List of users with read-only privileges +## @field database.extensions {[]string} Extensions enabled for the database +## ## Example: ## databases: ## myapp: @@ -64,27 +77,29 @@ databases: {} ## @section Backup parameters -## @param backup.enabled Enable regular backups -## @param backup.schedule Cron schedule for automated backups -## @param backup.retentionPolicy Retention policy -## @param backup.destinationPath Path to store the backup (i.e. s3://bucket/path/to/folder) -## @param backup.endpointURL S3 Endpoint used to upload data to the cloud -## @param backup.s3AccessKey Access key for S3, used for authentication -## @param backup.s3SecretKey Secret key for S3, used for authentication +## @param backup {backup} Backup configuration +## @field backup.enabled {bool} Enable regular backups +## @field backup.schedule {string} Cron schedule for automated backups +## @field backup.retentionPolicy {string} Retention policy +## @field backup.destinationPath {string} Path to store the backup (i.e. s3://bucket/path/to/folder) +## @field backup.endpointURL {string} S3 Endpoint used to upload data to the cloud +## @field backup.s3AccessKey {string} Access key for S3, used for authentication +## @field backup.s3SecretKey {string} Secret key for S3, used for authentication backup: enabled: false retentionPolicy: 30d - destinationPath: s3://bucket/path/to/folder/ - endpointURL: http://minio-gateway-service:9000 + destinationPath: "s3://bucket/path/to/folder/" + endpointURL: "http://minio-gateway-service:9000" schedule: "0 2 * * * *" - s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu - s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog + s3AccessKey: "" + s3SecretKey: "" ## @section Bootstrap (recovery) parameters -## @param bootstrap.enabled Restore database cluster from a backup -## @param bootstrap.recoveryTime Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest -## @param bootstrap.oldName Name of database cluster before deleting +## @param bootstrap {bootstrap} Bootstrap configuration +## @field bootstrap.enabled {bool} Restore database cluster from a backup +## @field bootstrap.recoveryTime {*string} Timestamp (PITR) up to which recovery will proceed, expressed in RFC 3339 format. If left empty, will restore latest +## @field bootstrap.oldName {string} Name of database cluster before deleting ## bootstrap: enabled: false diff --git a/packages/apps/virtual-machine/Makefile b/packages/apps/virtual-machine/Makefile index 5234bd85..6834018d 100644 --- a/packages/apps/virtual-machine/Makefile +++ b/packages/apps/virtual-machine/Makefile @@ -1,12 +1,9 @@ include ../../../scripts/package.mk generate: - readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md + cozyvalues-gen -v values.yaml -s values.schema.json -r README.md yq -o json -i '.properties.gpus.items.type = "object" | .properties.gpus.default = []' values.schema.json # INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \ # && yq -i -o json ".properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json PREFERENCES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/preferences.yaml | yq 'split(" ") | . + [""]' -o json) \ && yq -i -o json ".properties.instanceProfile.enum = $${PREFERENCES}" values.schema.json - yq -i -o json '.properties.externalPorts.items.type = "integer"' values.schema.json - yq -i -o json '.properties.systemDisk.properties.image.enum = ["ubuntu", "cirros", "alpine", "fedora", "talos"]' values.schema.json - yq -i -o json '.properties.externalMethod.enum = ["PortList", "WholeIP"]' values.schema.json diff --git a/packages/apps/virtual-machine/README.md b/packages/apps/virtual-machine/README.md index a1a89bdc..6ed30826 100644 --- a/packages/apps/virtual-machine/README.md +++ b/packages/apps/virtual-machine/README.md @@ -36,24 +36,28 @@ virtctl ssh @ ### Common parameters -| Name | Description | Value | -| ------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------ | -| `external` | Enable external access from outside the cluster | `false` | -| `externalMethod` | specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` | `PortList` | -| `externalPorts` | Specify ports to forward from outside the cluster | `[]` | -| `running` | Determines if the virtual machine should be running | `true` | -| `instanceType` | Virtual Machine instance type | `u1.medium` | -| `instanceProfile` | Virtual Machine preferences profile | `ubuntu` | -| `systemDisk.image` | The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos` | `ubuntu` | -| `systemDisk.storage` | The size of the disk allocated for the virtual machine | `5Gi` | -| `systemDisk.storageClass` | StorageClass used to store the data | `replicated` | -| `gpus` | List of GPUs to attach | `[]` | -| `resources.cpu` | The number of CPU cores allocated to the virtual machine | `""` | -| `resources.memory` | The amount of memory allocated to the virtual machine | `""` | -| `resources.sockets` | The number of CPU sockets allocated to the virtual machine (used to define vCPU topology) | `""` | -| `sshKeys` | List of SSH public keys for authentication. Can be a single key or a list of keys. | `[]` | -| `cloudInit` | cloud-init user data config. See cloud-init documentation for more details. | `""` | -| `cloudInitSeed` | A seed string to generate an SMBIOS UUID for the VM. | `""` | +| Name | Description | Type | Value | +| ------------------------- | ----------------------------------------------------------------------------------------------------------- | ---------- | ------------ | +| `external` | Enable external access from outside the cluster | `bool` | `false` | +| `externalMethod` | Specify method to pass through the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` | `string` | `{}` | +| `externalPorts` | Specify ports to forward from outside the cluster | `[]int` | `[22]` | +| `running` | if the virtual machine should be running | `bool` | `true` | +| `instanceType` | Virtual Machine instance type | `string` | `u1.medium` | +| `instanceProfile` | Virtual Machine preferences profile | `string` | `ubuntu` | +| `systemDisk` | System disk configuration | `object` | `{}` | +| `systemDisk.image` | The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos` | `string` | `ubuntu` | +| `systemDisk.storage` | The size of the disk allocated for the virtual machine | `string` | `5Gi` | +| `systemDisk.storageClass` | StorageClass used to store the data | `*string` | `replicated` | +| `gpus` | List of GPUs to attach | `[]object` | `[]` | +| `gpus[i].name` | The name of the GPU to attach. This should match the GPU resource name in the cluster. | `string` | `""` | +| `resources` | Resources | `object` | `{}` | +| `resources.cpu` | The number of CPU cores allocated to the virtual machine | `*string` | `null` | +| `resources.sockets` | The number of CPU sockets allocated to the virtual machine (used to define vCPU topology) | `*string` | `null` | +| `resources.memory` | The amount of memory allocated to the virtual machine | `*string` | `null` | +| `sshKeys` | List of SSH public keys for authentication. Can be a single key or a list of keys. | `[]string` | `[]` | +| `cloudInit` | cloud-init user data config. See cloud-init documentation for more details. | `string` | `""` | +| `cloudInitSeed` | A seed string to generate an SMBIOS UUID for the VM. | `string` | `""` | + ## U Series diff --git a/packages/apps/virtual-machine/values.schema.json b/packages/apps/virtual-machine/values.schema.json index 2d328886..999d60d6 100644 --- a/packages/apps/virtual-machine/values.schema.json +++ b/packages/apps/virtual-machine/values.schema.json @@ -1,49 +1,60 @@ { + "title": "Chart Values", + "type": "object", "properties": { "cloudInit": { - "default": "", "description": "cloud-init user data config. See cloud-init documentation for more details.", "type": "string" }, "cloudInitSeed": { - "default": "", "description": "A seed string to generate an SMBIOS UUID for the VM.", "type": "string" }, "external": { - "default": false, "description": "Enable external access from outside the cluster", - "type": "boolean" + "type": "boolean", + "default": false }, "externalMethod": { - "default": "PortList", - "description": "specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList`", + "description": "Specify method to pass through the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList`", "type": "string", + "default": "PortList", "enum": [ "PortList", "WholeIP" ] }, "externalPorts": { - "default": [], "description": "Specify ports to forward from outside the cluster", + "type": "array", + "default": [ + 22 + ], "items": { "type": "integer" - }, - "type": "array" + } }, "gpus": { - "default": [], "description": "List of GPUs to attach", + "type": "array", + "default": [], "items": { - "type": "object" - }, - "type": "array" + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "description": "The name of the GPU to attach. This should match the GPU resource name in the cluster.", + "type": "string" + } + } + } }, "instanceProfile": { - "default": "ubuntu", "description": "Virtual Machine preferences profile", "type": "string", + "default": "ubuntu", "enum": [ "alpine", "centos.7", @@ -91,47 +102,65 @@ ] }, "instanceType": { - "default": "u1.medium", "description": "Virtual Machine instance type", - "type": "string" + "type": "string", + "default": "u1.medium" }, "resources": { + "description": "Resources", + "type": "object", + "default": {}, "properties": { "cpu": { - "default": "", "description": "The number of CPU cores allocated to the virtual machine", - "type": "string" + "type": "string", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true }, "memory": { - "default": "", "description": "The amount of memory allocated to the virtual machine", - "type": "string" + "type": "string", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true }, "sockets": { - "default": "", "description": "The number of CPU sockets allocated to the virtual machine (used to define vCPU topology)", - "type": "string" + "type": "string", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true } - }, - "type": "object" + } }, "running": { - "default": true, - "description": "Determines if the virtual machine should be running", - "type": "boolean" + "description": "if the virtual machine should be running", + "type": "boolean", + "default": true }, "sshKeys": { - "default": [], "description": "List of SSH public keys for authentication. Can be a single key or a list of keys.", - "items": {}, - "type": "array" + "type": "array", + "default": [], + "items": { + "type": "string" + } }, "systemDisk": { + "description": "System disk configuration", + "type": "object", + "default": { + "image": "ubuntu", + "storage": "5Gi", + "storageClass": "replicated" + }, + "required": [ + "image", + "storage" + ], "properties": { "image": { - "default": "ubuntu", "description": "The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos`", "type": "string", + "default": "ubuntu", "enum": [ "ubuntu", "cirros", @@ -141,19 +170,16 @@ ] }, "storage": { - "default": "5Gi", "description": "The size of the disk allocated for the virtual machine", - "type": "string" + "type": "string", + "default": "5Gi" }, "storageClass": { - "default": "replicated", "description": "StorageClass used to store the data", - "type": "string" + "type": "string", + "default": "replicated" } - }, - "type": "object" + } } - }, - "title": "Chart Values", - "type": "object" + } } diff --git a/packages/apps/virtual-machine/values.yaml b/packages/apps/virtual-machine/values.yaml index 116e8ab3..f993a913 100644 --- a/packages/apps/virtual-machine/values.yaml +++ b/packages/apps/virtual-machine/values.yaml @@ -1,46 +1,53 @@ + ## @section Common parameters - -## @param external Enable external access from outside the cluster -## @param externalMethod specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` -## @param externalPorts [array] Specify ports to forward from outside the cluster +## +## @param external {bool} Enable external access from outside the cluster external: false -externalMethod: PortList +## @param externalMethod {string enum:"PortList,WholeIP"} Specify method to pass through the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` +externalMethod: "PortList" +## @param externalPorts {[]int} Specify ports to forward from outside the cluster externalPorts: -- 22 + - 22 -## @param running Determines if the virtual machine should be running +## @param running {bool} if the virtual machine should be running running: true -## @param instanceType Virtual Machine instance type -## @param instanceProfile Virtual Machine preferences profile +## @param instanceType {string} Virtual Machine instance type +## @param instanceProfile {string} Virtual Machine preferences profile ## instanceType: "u1.medium" instanceProfile: ubuntu -## @param systemDisk.image The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos` -## @param systemDisk.storage The size of the disk allocated for the virtual machine -## @param systemDisk.storageClass StorageClass used to store the data -## +## @param systemDisk {systemDisk} System disk configuration +## @field systemDisk.image {string enum:"ubuntu,cirros,alpine,fedora,talos"} The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos` +## @field systemDisk.storage {string} The size of the disk allocated for the virtual machine +## @field systemDisk.storageClass {*string} StorageClass used to store the data +## systemDisk: image: ubuntu storage: 5Gi storageClass: replicated -## @param gpus [array] List of GPUs to attach +## @param gpus {[]gpu} List of GPUs to attach +## @field gpu.name {string} The name of the GPU to attach. This should match the GPU resource name in the cluster. ## Example: ## gpus: ## - name: nvidia.com/GA102GL_A10 gpus: [] -## @param resources.cpu The number of CPU cores allocated to the virtual machine -## @param resources.memory The amount of memory allocated to the virtual machine -## @param resources.sockets The number of CPU sockets allocated to the virtual machine (used to define vCPU topology) -resources: - cpu: "" - memory: "" - sockets: "" -## @param sshKeys [array] List of SSH public keys for authentication. Can be a single key or a list of keys. +## @param resources {resources} Resources +## @field resources.cpu {*quantity} The number of CPU cores allocated to the virtual machine +## @field resources.sockets {*quantity} The number of CPU sockets allocated to the virtual machine (used to define vCPU topology) +## @field resources.memory {*quantity} The amount of memory allocated to the virtual machine +## Example: +## resources: +## cpu: "4" +## sockets: "1" +## memory: "8Gi" +resources: {} + +## @param sshKeys {[]string} List of SSH public keys for authentication. Can be a single key or a list of keys. ## Example: ## sshKeys: ## - ssh-rsa ... @@ -48,7 +55,7 @@ resources: ## sshKeys: [] -## @param cloudInit cloud-init user data config. See cloud-init documentation for more details. +## @param cloudInit {string} cloud-init user data config. See cloud-init documentation for more details. ## - https://cloudinit.readthedocs.io/en/latest/explanation/format.html ## - https://cloudinit.readthedocs.io/en/latest/reference/examples.html ## Example: @@ -59,11 +66,11 @@ sshKeys: [] ## cloudInit: "" -## @param cloudInitSeed A seed string to generate an SMBIOS UUID for the VM. -cloudInitSeed: "" +## @param cloudInitSeed {string} A seed string to generate an SMBIOS UUID for the VM. ## Change it to any new value to force a full cloud-init reconfiguration. Change it when you want to apply ## to an existing VM settings that are usually written only once, like new SSH keys or new network configuration. ## An empty value does nothing (and the existing UUID is not reverted). Please note that changing this value ## does not trigger a VM restart. You must perform the restart separately. ## Example: ## cloudInitSeed: "upd1" +cloudInitSeed: "" diff --git a/packages/extra/monitoring/Makefile b/packages/extra/monitoring/Makefile index 8b7e72da..53608e14 100644 --- a/packages/extra/monitoring/Makefile +++ b/packages/extra/monitoring/Makefile @@ -6,11 +6,7 @@ include ../../../scripts/common-envs.mk include ../../../scripts/package.mk generate: - readme-generator-for-helm -v values.yaml -s values.schema.json.tmp -r README.md - cat values.schema.json.tmp | \ - jq '.properties.metricsStorages.items.type = "object" | .properties.logsStorages.items.type = "object"' \ - > values.schema.json - rm -f values.schema.json.tmp + cozyvalues-gen -v values.yaml -s values.schema.json -r README.md image: docker buildx build images/grafana \ diff --git a/packages/extra/monitoring/README.md b/packages/extra/monitoring/README.md index 5fa84dce..7bf4b646 100644 --- a/packages/extra/monitoring/README.md +++ b/packages/extra/monitoring/README.md @@ -4,22 +4,88 @@ ### Common parameters -| Name | Description | Value | -| ----------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------- | -| `host` | The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host). | `""` | -| `metricsStorages` | Configuration of metrics storage instances | `[]` | -| `logsStorages` | Configuration of logs storage instances | `[]` | -| `alerta.storage` | Persistent Volume size for alerta database | `10Gi` | -| `alerta.storageClassName` | StorageClass used to store the data | `""` | -| `alerta.resources.requests.cpu` | The minimum amount of CPU required for alerta | `100m` | -| `alerta.resources.requests.memory` | The minimum amount of memory required for alerta | `256Mi` | -| `alerta.resources.limits.cpu` | The maximum amount of CPU allowed for alerta | `1` | -| `alerta.resources.limits.memory` | The maximum amount of memory allowed for alerta | `1Gi` | -| `alerta.alerts.telegram.token` | telegram token for your bot | `""` | -| `alerta.alerts.telegram.chatID` | specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot | `""` | -| `alerta.alerts.telegram.disabledSeverity` | list of severity without alerts, separated comma like: "informational,warning" | `""` | -| `grafana.db.size` | Persistent Volume size for grafana database | `10Gi` | -| `grafana.resources.requests.cpu` | The minimum amount of CPU required for grafana | `100m` | -| `grafana.resources.requests.memory` | The minimum amount of memory required for grafana | `256Mi` | -| `grafana.resources.limits.cpu` | The maximum amount of CPU allowed for grafana | `1` | -| `grafana.resources.limits.memory` | The maximum amount of memory allowed for grafana | `1Gi` | +| Name | Description | Type | Value | +| ------ | --------------------------------------------------------------------------------------------------------- | -------- | ----- | +| `host` | The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host). | `string` | `""` | + + +### Metrics storage configuration + +| Name | Description | Type | Value | +| ------------------------------------------------ | -------------------------------------------------------------- | ---------- | ------- | +| `metricsStorages` | Configuration of metrics storage instances | `[]object` | `[...]` | +| `metricsStorages[i].name` | Name of the storage instance | `string` | `""` | +| `metricsStorages[i].retentionPeriod` | Retention period for the metrics in the storage instance | `string` | `""` | +| `metricsStorages[i].deduplicationInterval` | Deduplication interval for the metrics in the storage instance | `string` | `""` | +| `metricsStorages[i].storage` | Persistent Volume size for the storage instance | `string` | `""` | +| `metricsStorages[i].storageClassName` | StorageClass used to store the data | `*string` | `null` | +| `metricsStorages[i].vminsert` | Configuration for vminsert component of the storage instance | `*object` | `null` | +| `metricsStorages[i].vminsert.minAllowed` | Requests (minimum allowed/available resources) | `*object` | `null` | +| `metricsStorages[i].vminsert.minAllowed.cpu` | CPU request (minimum available CPU) | `*string` | `null` | +| `metricsStorages[i].vminsert.minAllowed.memory` | Memory request (minimum available memory) | `*string` | `null` | +| `metricsStorages[i].vminsert.maxAllowed` | Limits (maximum allowed/available resources ) | `*object` | `null` | +| `metricsStorages[i].vminsert.maxAllowed.cpu` | CPU limit (maximum available CPU) | `*string` | `null` | +| `metricsStorages[i].vminsert.maxAllowed.memory` | Memory limit (maximum available memory) | `*string` | `null` | +| `metricsStorages[i].vmselect` | Configuration for vmselect component of the storage instance | `*object` | `null` | +| `metricsStorages[i].vmselect.minAllowed` | Requests (minimum allowed/available resources) | `*object` | `null` | +| `metricsStorages[i].vmselect.minAllowed.cpu` | CPU request (minimum available CPU) | `*string` | `null` | +| `metricsStorages[i].vmselect.minAllowed.memory` | Memory request (minimum available memory) | `*string` | `null` | +| `metricsStorages[i].vmselect.maxAllowed` | Limits (maximum allowed/available resources ) | `*object` | `null` | +| `metricsStorages[i].vmselect.maxAllowed.cpu` | CPU limit (maximum available CPU) | `*string` | `null` | +| `metricsStorages[i].vmselect.maxAllowed.memory` | Memory limit (maximum available memory) | `*string` | `null` | +| `metricsStorages[i].vmstorage` | Configuration for vmstorage component of the storage instance | `*object` | `null` | +| `metricsStorages[i].vmstorage.minAllowed` | Requests (minimum allowed/available resources) | `*object` | `null` | +| `metricsStorages[i].vmstorage.minAllowed.cpu` | CPU request (minimum available CPU) | `*string` | `null` | +| `metricsStorages[i].vmstorage.minAllowed.memory` | Memory request (minimum available memory) | `*string` | `null` | +| `metricsStorages[i].vmstorage.maxAllowed` | Limits (maximum allowed/available resources ) | `*object` | `null` | +| `metricsStorages[i].vmstorage.maxAllowed.cpu` | CPU limit (maximum available CPU) | `*string` | `null` | +| `metricsStorages[i].vmstorage.maxAllowed.memory` | Memory limit (maximum available memory) | `*string` | `null` | + + +### Logs storage configuration + +| Name | Description | Type | Value | +| ---------------------------------- | ----------------------------------------------------- | ---------- | ------- | +| `logsStorages` | Configuration of logs storage instances | `[]object` | `[...]` | +| `logsStorages[i].name` | Name of the storage instance | `string` | `""` | +| `logsStorages[i].retentionPeriod` | Retention period for the logs in the storage instance | `string` | `""` | +| `logsStorages[i].storage` | Persistent Volume size for the storage instance | `string` | `""` | +| `logsStorages[i].storageClassName` | StorageClass used to store the data | `*string` | `null` | + + +### Alerta configuration + +| Name | Description | Type | Value | +| ----------------------------------------- | ----------------------------------------------------------------------------------- | --------- | ------- | +| `alerta` | Configuration for Alerta service | `object` | `{}` | +| `alerta.storage` | Persistent Volume size for the database | `string` | `10Gi` | +| `alerta.storageClassName` | StorageClass used to store the data | `string` | `""` | +| `alerta.resources` | Resources configuration | `*object` | `null` | +| `alerta.resources.requests` | | `*object` | `null` | +| `alerta.resources.requests.cpu` | CPU request (minimum available CPU) | `*string` | `100m` | +| `alerta.resources.requests.memory` | Memory request (minimum available memory) | `*string` | `256Mi` | +| `alerta.resources.limits` | | `*object` | `null` | +| `alerta.resources.limits.cpu` | CPU limit (maximum available CPU) | `*string` | `1` | +| `alerta.resources.limits.memory` | Memory limit (maximum available memory) | `*string` | `1Gi` | +| `alerta.alerts` | Configuration for alerts | `object` | `{}` | +| `alerta.alerts.telegram` | Configuration for Telegram alerts | `object` | `{}` | +| `alerta.alerts.telegram.token` | Telegram token for your bot | `string` | `""` | +| `alerta.alerts.telegram.chatID` | Specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot | `string` | `""` | +| `alerta.alerts.telegram.disabledSeverity` | List of severity without alerts, separated by comma like: "informational,warning" | `string` | `""` | + + +### Grafana configuration + +| Name | Description | Type | Value | +| ----------------------------------- | ----------------------------------------- | --------- | ------- | +| `grafana` | Configuration for Grafana | `object` | `{}` | +| `grafana.db` | Database configuration | `object` | `{}` | +| `grafana.db.size` | Persistent Volume size for the database | `string` | `10Gi` | +| `grafana.resources` | Resources configuration | `*object` | `null` | +| `grafana.resources.requests` | | `*object` | `null` | +| `grafana.resources.requests.cpu` | CPU request (minimum available CPU) | `*string` | `100m` | +| `grafana.resources.requests.memory` | Memory request (minimum available memory) | `*string` | `256Mi` | +| `grafana.resources.limits` | | `*object` | `null` | +| `grafana.resources.limits.cpu` | CPU limit (maximum available CPU) | `*string` | `1` | +| `grafana.resources.limits.memory` | Memory limit (maximum available memory) | `*string` | `1Gi` | + diff --git a/packages/extra/monitoring/values.schema.json b/packages/extra/monitoring/values.schema.json index 75f76eb5..ae807aa8 100644 --- a/packages/extra/monitoring/values.schema.json +++ b/packages/extra/monitoring/values.schema.json @@ -1,152 +1,487 @@ { + "title": "Chart Values", + "type": "object", "properties": { "alerta": { + "description": "Configuration for Alerta service", + "type": "object", + "default": { + "alerts": { + "telegram": { + "chatID": "", + "disabledSeverity": "", + "token": "" + } + }, + "resources": { + "limits": { + "cpu": "1", + "memory": "1Gi" + }, + "requests": { + "cpu": "100m", + "memory": "256Mi" + } + }, + "storage": "10Gi", + "storageClassName": "" + }, + "required": [ + "alerts", + "storage", + "storageClassName" + ], "properties": { "alerts": { + "description": "Configuration for alerts", + "type": "object", + "default": { + "telegram": { + "chatID": "", + "disabledSeverity": "", + "token": "" + } + }, + "required": [ + "telegram" + ], "properties": { "telegram": { + "description": "Configuration for Telegram alerts", + "type": "object", + "default": { + "chatID": "", + "disabledSeverity": "", + "token": "" + }, + "required": [ + "chatID", + "disabledSeverity", + "token" + ], "properties": { "chatID": { - "default": "", - "description": "specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot", + "description": "Specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot", "type": "string" }, "disabledSeverity": { - "default": "", - "description": "list of severity without alerts, separated comma like: \"informational,warning\"", + "description": "List of severity without alerts, separated by comma like: \"informational,warning\"", "type": "string" }, "token": { - "default": "", - "description": "telegram token for your bot", + "description": "Telegram token for your bot", "type": "string" } - }, - "type": "object" + } } - }, - "type": "object" + } }, "resources": { - "properties": { + "description": "Resources configuration", + "type": "object", + "default": { "limits": { - "properties": { - "cpu": { - "default": "1", - "description": "The maximum amount of CPU allowed for alerta", - "type": "string" - }, - "memory": { - "default": "1Gi", - "description": "The maximum amount of memory allowed for alerta", - "type": "string" - } - }, - "type": "object" + "cpu": "1", + "memory": "1Gi" }, "requests": { - "properties": { - "cpu": { - "default": "100m", - "description": "The minimum amount of CPU required for alerta", - "type": "string" - }, - "memory": { - "default": "256Mi", - "description": "The minimum amount of memory required for alerta", - "type": "string" - } - }, - "type": "object" + "cpu": "100m", + "memory": "256Mi" } }, - "type": "object" + "properties": { + "limits": { + "type": "object", + "default": { + "cpu": "1", + "memory": "1Gi" + }, + "properties": { + "cpu": { + "description": "CPU limit (maximum available CPU)", + "type": "string", + "default": "1", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory limit (maximum available memory)", + "type": "string", + "default": "1Gi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + }, + "requests": { + "type": "object", + "default": { + "cpu": "100m", + "memory": "256Mi" + }, + "properties": { + "cpu": { + "description": "CPU request (minimum available CPU)", + "type": "string", + "default": "100m", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory request (minimum available memory)", + "type": "string", + "default": "256Mi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + } + } }, "storage": { - "default": "10Gi", - "description": "Persistent Volume size for alerta database", - "type": "string" + "description": "Persistent Volume size for the database", + "type": "string", + "default": "10Gi" }, "storageClassName": { - "default": "", "description": "StorageClass used to store the data", "type": "string" } - }, - "type": "object" + } }, "grafana": { - "properties": { + "description": "Configuration for Grafana", + "type": "object", + "default": { "db": { - "properties": { - "size": { - "default": "10Gi", - "description": "Persistent Volume size for grafana database", - "type": "string" - } - }, - "type": "object" + "size": "10Gi" }, "resources": { - "properties": { - "limits": { - "properties": { - "cpu": { - "default": "1", - "description": "The maximum amount of CPU allowed for grafana", - "type": "string" - }, - "memory": { - "default": "1Gi", - "description": "The maximum amount of memory allowed for grafana", - "type": "string" - } - }, - "type": "object" - }, - "requests": { - "properties": { - "cpu": { - "default": "100m", - "description": "The minimum amount of CPU required for grafana", - "type": "string" - }, - "memory": { - "default": "256Mi", - "description": "The minimum amount of memory required for grafana", - "type": "string" - } - }, - "type": "object" - } + "limits": { + "cpu": "1", + "memory": "1Gi" }, - "type": "object" + "requests": { + "cpu": "100m", + "memory": "256Mi" + } } }, - "type": "object" + "required": [ + "db" + ], + "properties": { + "db": { + "description": "Database configuration", + "type": "object", + "default": { + "size": "10Gi" + }, + "required": [ + "size" + ], + "properties": { + "size": { + "description": "Persistent Volume size for the database", + "type": "string", + "default": "10Gi" + } + } + }, + "resources": { + "description": "Resources configuration", + "type": "object", + "default": { + "limits": { + "cpu": "1", + "memory": "1Gi" + }, + "requests": { + "cpu": "100m", + "memory": "256Mi" + } + }, + "properties": { + "limits": { + "type": "object", + "default": { + "cpu": "1", + "memory": "1Gi" + }, + "properties": { + "cpu": { + "description": "CPU limit (maximum available CPU)", + "type": "string", + "default": "1", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory limit (maximum available memory)", + "type": "string", + "default": "1Gi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + }, + "requests": { + "type": "object", + "default": { + "cpu": "100m", + "memory": "256Mi" + }, + "properties": { + "cpu": { + "description": "CPU request (minimum available CPU)", + "type": "string", + "default": "100m", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory request (minimum available memory)", + "type": "string", + "default": "256Mi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + } + } + } + } }, "host": { - "default": "", "description": "The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host).", "type": "string" }, "logsStorages": { - "default": [], "description": "Configuration of logs storage instances", + "type": "array", + "default": [ + { + "name": "generic", + "retentionPeriod": "1", + "storage": "10Gi", + "storageClassName": "replicated" + } + ], "items": { - "type": "object" - }, - "type": "array" + "type": "object", + "required": [ + "name", + "retentionPeriod", + "storage" + ], + "properties": { + "name": { + "description": "Name of the storage instance", + "type": "string" + }, + "retentionPeriod": { + "description": "Retention period for the logs in the storage instance", + "type": "string" + }, + "storage": { + "description": "Persistent Volume size for the storage instance", + "type": "string" + }, + "storageClassName": { + "description": "StorageClass used to store the data", + "type": "string" + } + } + } }, "metricsStorages": { - "default": [], "description": "Configuration of metrics storage instances", + "type": "array", + "default": [ + { + "deduplicationInterval": "15s", + "name": "shortterm", + "retentionPeriod": "3d", + "storage": "10Gi", + "storageClassName": "" + }, + { + "deduplicationInterval": "5m", + "name": "longterm", + "retentionPeriod": "14d", + "storage": "10Gi", + "storageClassName": "" + } + ], "items": { - "type": "object" - }, - "type": "array" + "type": "object", + "required": [ + "deduplicationInterval", + "name", + "retentionPeriod", + "storage" + ], + "properties": { + "deduplicationInterval": { + "description": "Deduplication interval for the metrics in the storage instance", + "type": "string" + }, + "name": { + "description": "Name of the storage instance", + "type": "string" + }, + "retentionPeriod": { + "description": "Retention period for the metrics in the storage instance", + "type": "string" + }, + "storage": { + "description": "Persistent Volume size for the storage instance", + "type": "string" + }, + "storageClassName": { + "description": "StorageClass used to store the data", + "type": "string" + }, + "vminsert": { + "description": "Configuration for vminsert component of the storage instance", + "type": "object", + "properties": { + "maxAllowed": { + "description": "Limits (maximum allowed/available resources )", + "type": "object", + "properties": { + "cpu": { + "description": "CPU limit (maximum available CPU)", + "type": "string", + "default": "1", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory limit (maximum available memory)", + "type": "string", + "default": "1Gi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + }, + "minAllowed": { + "description": "Requests (minimum allowed/available resources)", + "type": "object", + "properties": { + "cpu": { + "description": "CPU request (minimum available CPU)", + "type": "string", + "default": "100m", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory request (minimum available memory)", + "type": "string", + "default": "256Mi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + } + } + }, + "vmselect": { + "description": "Configuration for vmselect component of the storage instance", + "type": "object", + "properties": { + "maxAllowed": { + "description": "Limits (maximum allowed/available resources )", + "type": "object", + "properties": { + "cpu": { + "description": "CPU limit (maximum available CPU)", + "type": "string", + "default": "1", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory limit (maximum available memory)", + "type": "string", + "default": "1Gi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + }, + "minAllowed": { + "description": "Requests (minimum allowed/available resources)", + "type": "object", + "properties": { + "cpu": { + "description": "CPU request (minimum available CPU)", + "type": "string", + "default": "100m", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory request (minimum available memory)", + "type": "string", + "default": "256Mi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + } + } + }, + "vmstorage": { + "description": "Configuration for vmstorage component of the storage instance", + "type": "object", + "properties": { + "maxAllowed": { + "description": "Limits (maximum allowed/available resources )", + "type": "object", + "properties": { + "cpu": { + "description": "CPU limit (maximum available CPU)", + "type": "string", + "default": "1", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory limit (maximum available memory)", + "type": "string", + "default": "1Gi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + }, + "minAllowed": { + "description": "Requests (minimum allowed/available resources)", + "type": "object", + "properties": { + "cpu": { + "description": "CPU request (minimum available CPU)", + "type": "string", + "default": "100m", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + }, + "memory": { + "description": "Memory request (minimum available memory)", + "type": "string", + "default": "256Mi", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + } + } + } + } + } + } } - }, - "title": "Chart Values", - "type": "object" -} + } +} \ No newline at end of file diff --git a/packages/extra/monitoring/values.yaml b/packages/extra/monitoring/values.yaml index 9e992f22..f507ba05 100644 --- a/packages/extra/monitoring/values.yaml +++ b/packages/extra/monitoring/values.yaml @@ -1,10 +1,29 @@ ## @section Common parameters -## @param host The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host). +## @param host {string} The hostname used to access the grafana externally (defaults to 'grafana' subdomain for the tenant host). host: "" -## @param metricsStorages [array] Configuration of metrics storage instances -## +## @section Metrics storage configuration + +## @param metricsStorages {[]metricsStorage} Configuration of metrics storage instances +## @field metricsStorage.name {string} Name of the storage instance +## @field metricsStorage.retentionPeriod {string} Retention period for the metrics in the storage instance +## @field metricsStorage.deduplicationInterval {string} Deduplication interval for the metrics in the storage instance +## @field metricsStorage.storage {string} Persistent Volume size for the storage instance +## @field metricsStorage.storageClassName {*string} StorageClass used to store the data +## @field metricsStorage.vminsert {*vmcomponent} Configuration for vminsert component of the storage instance +## @field metricsStorage.vmselect {*vmcomponent} Configuration for vmselect component of the storage instance +## @field metricsStorage.vmstorage {*vmcomponent} Configuration for vmstorage component of the storage instance + +## @field request.cpu {*quantity} CPU request (minimum available CPU) +## @field request.memory {*quantity} Memory request (minimum available memory) +## @field limit.cpu {*quantity} CPU limit (maximum available CPU) +## @field limit.memory {*quantity} Memory limit (maximum available memory) +## @field vmcomponent.minAllowed {*request} Requests (minimum allowed/available resources) +## @field vmcomponent.maxAllowed {*limit} Limits (maximum allowed/available resources ) +## @field resources.requests {*request} +## @field resources.limits {*limit} + ## Example: ## metricsStorages: ## - name: shortterm @@ -46,7 +65,13 @@ metricsStorages: storage: 10Gi storageClassName: "" -## @param logsStorages [array] Configuration of logs storage instances +## @section Logs storage configuration + +## @param logsStorages {[]logsStorage} Configuration of logs storage instances +## @field logsStorage.name {string} Name of the storage instance +## @field logsStorage.retentionPeriod {string} Retention period for the logs in the storage instance +## @field logsStorage.storage {string} Persistent Volume size for the storage instance +## @field logsStorage.storageClassName {*string} StorageClass used to store the data ## logsStorages: - name: generic @@ -54,14 +79,17 @@ logsStorages: storage: 10Gi storageClassName: replicated -## Configuration for Alerta -## @param alerta.storage Persistent Volume size for alerta database -## @param alerta.storageClassName StorageClass used to store the data -## @param alerta.resources.requests.cpu The minimum amount of CPU required for alerta -## @param alerta.resources.requests.memory The minimum amount of memory required for alerta -## @param alerta.resources.limits.cpu The maximum amount of CPU allowed for alerta -## @param alerta.resources.limits.memory The maximum amount of memory allowed for alerta -## +## @section Alerta configuration + +## @param alerta {alerta} Configuration for Alerta service +## @field alerta.storage {string} Persistent Volume size for the database +## @field alerta.storageClassName {string} StorageClass used to store the data +## @field alerta.resources {*resources} Resources configuration +## @field alerta.alerts {alerts} Configuration for alerts +## @field alerts.telegram {telegramAlerts} Configuration for Telegram alerts +## @field telegramAlerts.token {string} Telegram token for your bot +## @field telegramAlerts.chatID {string} Specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot +## @field telegramAlerts.disabledSeverity {string} List of severity without alerts, separated by comma like: "informational,warning" alerta: storage: 10Gi storageClassName: "" @@ -73,9 +101,6 @@ alerta: cpu: 100m memory: 256Mi alerts: - ## @param alerta.alerts.telegram.token telegram token for your bot - ## @param alerta.alerts.telegram.chatID specify multiple ID's separated by comma. Get yours in https://t.me/chatid_echo_bot - ## @param alerta.alerts.telegram.disabledSeverity list of severity without alerts, separated comma like: "informational,warning" ## example: ## telegram: ## token: "7262461387:AAGtwq16iwuVtWtzoN6TUEMpF00fpC9Xz34" @@ -87,12 +112,14 @@ alerta: chatID: "" disabledSeverity: "" -## Configuration for Grafana -## @param grafana.db.size Persistent Volume size for grafana database -## @param grafana.resources.requests.cpu The minimum amount of CPU required for grafana -## @param grafana.resources.requests.memory The minimum amount of memory required for grafana -## @param grafana.resources.limits.cpu The maximum amount of CPU allowed for grafana -## @param grafana.resources.limits.memory The maximum amount of memory allowed for grafana +## @section Grafana configuration + +## @param grafana {grafana} Configuration for Grafana +## @field grafana.db {grafanaDB} Database configuration +## @field grafanaDB.size {string} Persistent Volume size for the database +## @field grafana.resources {*resources} Resources configuration + + grafana: db: size: 10Gi From 6e119ba940f54c76b3b14c3ec0ee7a5212e9a5e6 Mon Sep 17 00:00:00 2001 From: Kingdon B Date: Wed, 6 Aug 2025 13:25:19 -0400 Subject: [PATCH 28/30] Update Flux Operator to 0.27.0 Signed-off-by: Kingdon B --- .../charts/flux-operator/Chart.yaml | 4 +- .../charts/flux-operator/README.md | 4 +- .../charts/flux-operator/templates/crds.yaml | 10 + .../charts/flux-operator/values.schema.json | 193 +++++----- .../charts/flux-operator/values.yaml | 2 +- .../fluxcd/charts/flux-instance/Chart.yaml | 4 +- .../fluxcd/charts/flux-instance/README.md | 6 +- .../flux-instance/templates/healthcheck.yaml | 78 ++++ .../flux-instance/templates/instance.yaml | 7 +- .../templates/serviceaccount.yaml | 17 + .../charts/flux-instance/values.schema.json | 341 +++++++++++++++--- .../fluxcd/charts/flux-instance/values.yaml | 96 ++++- 12 files changed, 614 insertions(+), 148 deletions(-) create mode 100644 packages/system/fluxcd/charts/flux-instance/templates/healthcheck.yaml create mode 100644 packages/system/fluxcd/charts/flux-instance/templates/serviceaccount.yaml diff --git a/packages/system/fluxcd-operator/charts/flux-operator/Chart.yaml b/packages/system/fluxcd-operator/charts/flux-operator/Chart.yaml index 22fa307b..6c3384fe 100644 --- a/packages/system/fluxcd-operator/charts/flux-operator/Chart.yaml +++ b/packages/system/fluxcd-operator/charts/flux-operator/Chart.yaml @@ -8,7 +8,7 @@ annotations: - name: Upstream Project url: https://github.com/controlplaneio-fluxcd/flux-operator apiVersion: v2 -appVersion: v0.24.1 +appVersion: v0.27.0 description: 'A Helm chart for deploying the Flux Operator. ' home: https://github.com/controlplaneio-fluxcd icon: https://raw.githubusercontent.com/cncf/artwork/main/projects/flux/icon/color/flux-icon-color.png @@ -25,4 +25,4 @@ sources: - https://github.com/controlplaneio-fluxcd/flux-operator - https://github.com/controlplaneio-fluxcd/charts type: application -version: 0.24.1 +version: 0.27.0 diff --git a/packages/system/fluxcd-operator/charts/flux-operator/README.md b/packages/system/fluxcd-operator/charts/flux-operator/README.md index aea5906d..7622db10 100644 --- a/packages/system/fluxcd-operator/charts/flux-operator/README.md +++ b/packages/system/fluxcd-operator/charts/flux-operator/README.md @@ -1,6 +1,6 @@ # flux-operator -![Version: 0.24.1](https://img.shields.io/badge/Version-0.24.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.24.1](https://img.shields.io/badge/AppVersion-v0.24.1-informational?style=flat-square) +![Version: 0.27.0](https://img.shields.io/badge/Version-0.27.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.27.0](https://img.shields.io/badge/AppVersion-v0.27.0-informational?style=flat-square) The [Flux Operator](https://github.com/controlplaneio-fluxcd/flux-operator) provides a declarative API for the installation and upgrade of CNCF [Flux](https://fluxcd.io) and the @@ -56,7 +56,7 @@ see the Flux Operator [documentation](https://fluxcd.control-plane.io/operator/) | rbac.createAggregation | bool | `true` | Grant the Kubernetes view, edit and admin roles access to ResourceSet APIs. | | readinessProbe | object | `{"httpGet":{"path":"/readyz","port":8081},"initialDelaySeconds":5,"periodSeconds":10}` | Container readiness probe settings. | | reporting | object | `{"interval":"5m"}` | Flux [reporting](https://fluxcd.control-plane.io/operator/fluxreport/) settings. | -| resources | object | `{"limits":{"cpu":"1000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"64Mi"}}` | Container resources requests and limits settings. | +| resources | object | `{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"64Mi"}}` | Container resources requests and limits settings. | | securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Container security context settings. The default is compliant with the pod security restricted profile. | | serviceAccount | object | `{"automount":true,"create":true,"name":""}` | Pod service account settings. The name of the service account defaults to the release name. | | serviceMonitor | object | `{"create":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Prometheus Operator scraping settings. | diff --git a/packages/system/fluxcd-operator/charts/flux-operator/templates/crds.yaml b/packages/system/fluxcd-operator/charts/flux-operator/templates/crds.yaml index 9487a4a3..c00cf271 100644 --- a/packages/system/fluxcd-operator/charts/flux-operator/templates/crds.yaml +++ b/packages/system/fluxcd-operator/charts/flux-operator/templates/crds.yaml @@ -85,6 +85,16 @@ spec: required for object-level workload identity. This feature is only available in Flux v2.6.0 and later. type: boolean + size: + description: |- + Size defines the vertical scaling profile of the Flux controllers. + The size is used to determine the concurrency and CPU/Memory limits for the Flux controllers. + Accepted values are: 'small', 'medium' and 'large'. + enum: + - small + - medium + - large + type: string tenantDefaultServiceAccount: description: |- TenantDefaultServiceAccount is the name of the service account diff --git a/packages/system/fluxcd-operator/charts/flux-operator/values.schema.json b/packages/system/fluxcd-operator/charts/flux-operator/values.schema.json index dc277e55..bc62fa83 100644 --- a/packages/system/fluxcd-operator/charts/flux-operator/values.schema.json +++ b/packages/system/fluxcd-operator/charts/flux-operator/values.schema.json @@ -1,5 +1,10 @@ { "$schema": "https://json-schema.org/draft/2019-09/schema", + "type": "object", + "required": [ + "resources", + "securityContext" + ], "properties": { "affinity": { "default": { @@ -21,16 +26,23 @@ } } }, + "type": "object", "properties": { "nodeAffinity": { + "type": "object", "properties": { "requiredDuringSchedulingIgnoredDuringExecution": { + "type": "object", "properties": { "nodeSelectorTerms": { + "type": "array", "items": { + "type": "object", "properties": { "matchExpressions": { + "type": "array", "items": { + "type": "object", "properties": { "key": { "type": "string" @@ -39,29 +51,22 @@ "type": "string" }, "values": { + "type": "array", "items": { "type": "string" - }, - "type": "array" + } } - }, - "type": "object" - }, - "type": "array" + } + } } - }, - "type": "object" - }, - "type": "array" + } + } } - }, - "type": "object" + } } - }, - "type": "object" + } } - }, - "type": "object" + } }, "apiPriority": { "default": { @@ -69,6 +74,7 @@ "extraServiceAccounts": [], "level": "workload-high" }, + "type": "object", "properties": { "enabled": { "type": "boolean" @@ -79,30 +85,41 @@ "level": { "type": "string" } - }, - "type": "object" + } }, "commonAnnotations": { - "properties": {}, "type": "object" }, "commonLabels": { - "properties": {}, "type": "object" }, "extraArgs": { + "type": "array", + "uniqueItems": true, "items": { "type": "string" - }, - "type": "array", - "uniqueItems": true + } }, "extraEnvs": { + "type": "array", + "uniqueItems": true, "items": { "type": "object" - }, + } + }, + "extraVolumeMounts": { "type": "array", - "uniqueItems": true + "uniqueItems": true, + "items": { + "type": "object" + } + }, + "extraVolumes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object" + } }, "fullnameOverride": { "type": "string" @@ -112,21 +129,25 @@ "type": "boolean" }, "image": { + "type": "object", + "required": [ + "repository" + ], "properties": { "imagePullPolicy": { + "type": "string", "enum": [ "IfNotPresent", "Always", "Never" - ], - "type": "string" + ] }, "pullSecrets": { + "type": "array", + "uniqueItems": true, "items": { "type": "object" - }, - "type": "array", - "uniqueItems": true + } }, "repository": { "type": "string" @@ -134,11 +155,7 @@ "tag": { "type": "string" } - }, - "required": [ - "repository" - ], - "type": "object" + } }, "installCRDs": { "default": true, @@ -153,8 +170,10 @@ "initialDelaySeconds": 15, "periodSeconds": 20 }, + "type": "object", "properties": { "httpGet": { + "type": "object", "properties": { "path": { "type": "string" @@ -162,8 +181,7 @@ "port": { "type": "integer" } - }, - "type": "object" + } }, "initialDelaySeconds": { "type": "integer" @@ -171,18 +189,18 @@ "periodSeconds": { "type": "integer" } - }, - "type": "object" + } }, "logLevel": { + "type": "string", "enum": [ "debug", "info", "error" - ], - "type": "string" + ] }, "marketplace": { + "type": "object", "properties": { "account": { "type": "string" @@ -193,10 +211,13 @@ "type": { "type": "string" } - }, - "type": "object" + } }, "multitenancy": { + "type": "object", + "required": [ + "defaultServiceAccount" + ], "properties": { "defaultServiceAccount": { "type": "string" @@ -204,26 +225,18 @@ "enabled": { "type": "boolean" } - }, - "required": [ - "defaultServiceAccount" - ], - "type": "object" + } }, "nameOverride": { "type": "string" }, "nodeSelector": { - "properties": {}, - "type": [ - "object" - ] + "type": "object" }, "podSecurityContext": { "default": { "fsGroup": 1337 }, - "properties": {}, "type": "object" }, "priorityClassName": { @@ -231,6 +244,7 @@ "type": "string" }, "rbac": { + "type": "object", "properties": { "create": { "type": "boolean" @@ -238,8 +252,7 @@ "createAggregation": { "type": "boolean" } - }, - "type": "object" + } }, "readinessProbe": { "default": { @@ -250,8 +263,10 @@ "initialDelaySeconds": 5, "periodSeconds": 10 }, + "type": "object", "properties": { "httpGet": { + "type": "object", "properties": { "path": { "type": "string" @@ -259,8 +274,7 @@ "port": { "type": "integer" } - }, - "type": "object" + } }, "initialDelaySeconds": { "type": "integer" @@ -268,23 +282,24 @@ "periodSeconds": { "type": "integer" } - }, - "type": "object" + } }, "reporting": { + "type": "object", + "required": [ + "interval" + ], "properties": { "interval": { "type": "string" } - }, - "required": [ - "interval" - ], - "type": "object" + } }, "resources": { + "type": "object", "properties": { "limits": { + "type": "object", "properties": { "cpu": { "type": "string" @@ -292,14 +307,14 @@ "memory": { "type": "string" } - }, - "type": "object" + } }, "requests": { "default": { "cpu": "100m", "memory": "64Mi" }, + "type": "object", "properties": { "cpu": { "type": "string" @@ -307,13 +322,12 @@ "memory": { "type": "string" } - }, - "type": "object" + } } - }, - "type": "object" + } }, "securityContext": { + "type": "object", "properties": { "allowPrivilegeEscalation": { "default": false, @@ -325,16 +339,16 @@ "ALL" ] }, + "type": "object", "properties": { "drop": { + "type": "array", + "uniqueItems": true, "items": { "type": "string" - }, - "type": "array", - "uniqueItems": true + } } - }, - "type": "object" + } }, "readOnlyRootFilesystem": { "default": true, @@ -348,15 +362,14 @@ "default": { "type": "RuntimeDefault" }, + "type": "object", "properties": { "type": { "type": "string" } - }, - "type": "object" + } } - }, - "type": "object" + } }, "serviceAccount": { "default": { @@ -364,6 +377,7 @@ "create": true, "name": "" }, + "type": "object", "properties": { "automount": { "type": "boolean" @@ -374,8 +388,7 @@ "name": { "type": "string" } - }, - "type": "object" + } }, "serviceMonitor": { "default": { @@ -383,6 +396,7 @@ "interval": "60s", "scrapeTimeout": "30s" }, + "type": "object", "properties": { "create": { "type": "boolean" @@ -391,26 +405,19 @@ "type": "string" }, "labels": { - "properties": {}, "type": "object" }, "scrapeTimeout": { "type": "string" } - }, - "type": "object" + } }, "tolerations": { + "type": "array", + "uniqueItems": true, "items": { "type": "object" - }, - "type": "array", - "uniqueItems": true + } } - }, - "required": [ - "resources", - "securityContext" - ], - "type": "object" + } } diff --git a/packages/system/fluxcd-operator/charts/flux-operator/values.yaml b/packages/system/fluxcd-operator/charts/flux-operator/values.yaml index 7cbc14be..42c131b7 100644 --- a/packages/system/fluxcd-operator/charts/flux-operator/values.yaml +++ b/packages/system/fluxcd-operator/charts/flux-operator/values.yaml @@ -46,7 +46,7 @@ apiPriority: # @schema default: {"enabled":false,"level":"workload-high","extraS # -- Container resources requests and limits settings. resources: # @schema required: true limits: - cpu: 1000m + cpu: 2000m memory: 1Gi requests: # @schema default: {"cpu":"100m","memory":"64Mi"} cpu: 100m diff --git a/packages/system/fluxcd/charts/flux-instance/Chart.yaml b/packages/system/fluxcd/charts/flux-instance/Chart.yaml index 6b5c212d..b8f9cec4 100644 --- a/packages/system/fluxcd/charts/flux-instance/Chart.yaml +++ b/packages/system/fluxcd/charts/flux-instance/Chart.yaml @@ -8,7 +8,7 @@ annotations: - name: Upstream Project url: https://github.com/controlplaneio-fluxcd/flux-operator apiVersion: v2 -appVersion: v0.24.1 +appVersion: v0.27.0 description: 'A Helm chart for deploying a Flux instance managed by Flux Operator. ' home: https://github.com/controlplaneio-fluxcd icon: https://raw.githubusercontent.com/cncf/artwork/main/projects/flux/icon/color/flux-icon-color.png @@ -25,4 +25,4 @@ sources: - https://github.com/controlplaneio-fluxcd/flux-operator - https://github.com/controlplaneio-fluxcd/charts type: application -version: 0.24.1 +version: 0.27.0 diff --git a/packages/system/fluxcd/charts/flux-instance/README.md b/packages/system/fluxcd/charts/flux-instance/README.md index e73d44f6..315c184e 100644 --- a/packages/system/fluxcd/charts/flux-instance/README.md +++ b/packages/system/fluxcd/charts/flux-instance/README.md @@ -1,6 +1,6 @@ # flux-instance -![Version: 0.24.1](https://img.shields.io/badge/Version-0.24.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.24.1](https://img.shields.io/badge/AppVersion-v0.24.1-informational?style=flat-square) +![Version: 0.27.0](https://img.shields.io/badge/Version-0.27.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.27.0](https://img.shields.io/badge/AppVersion-v0.27.0-informational?style=flat-square) This chart is a thin wrapper around the `FluxInstance` custom resource, which is used by the [Flux Operator](https://github.com/controlplaneio-fluxcd/flux-operator) @@ -37,7 +37,9 @@ helm -n flux-system uninstall flux | commonAnnotations | object | `{}` | Common annotations to add to all deployed objects including pods. | | commonLabels | object | `{}` | Common labels to add to all deployed objects including pods. | | fullnameOverride | string | `"flux"` | | -| instance.cluster | object | `{"domain":"cluster.local","multitenant":false,"networkPolicy":true,"tenantDefaultServiceAccount":"default","type":"kubernetes"}` | Cluster https://fluxcd.control-plane.io/operator/fluxinstance/#cluster-configuration | +| healthcheck.enabled | bool | `false` | Enable post-install and post-upgrade health checks. | +| healthcheck.timeout | string | `"5m"` | Health check timeout in Go duration format. | +| instance.cluster | object | `{"domain":"cluster.local","multitenant":false,"networkPolicy":true,"size":"","tenantDefaultServiceAccount":"default","type":"kubernetes"}` | Cluster https://fluxcd.control-plane.io/operator/fluxinstance/#cluster-configuration | | instance.commonMetadata | object | `{"annotations":{},"labels":{}}` | Common metadata https://fluxcd.control-plane.io/operator/fluxinstance/#common-metadata | | instance.components | list | `["source-controller","kustomize-controller","helm-controller","notification-controller"]` | Components https://fluxcd.control-plane.io/operator/fluxinstance/#components-configuration | | instance.distribution | object | `{"artifact":"oci://ghcr.io/controlplaneio-fluxcd/flux-operator-manifests:latest","artifactPullSecret":"","imagePullSecret":"","registry":"ghcr.io/fluxcd","version":"2.x"}` | Distribution https://fluxcd.control-plane.io/operator/fluxinstance/#distribution-configuration | diff --git a/packages/system/fluxcd/charts/flux-instance/templates/healthcheck.yaml b/packages/system/fluxcd/charts/flux-instance/templates/healthcheck.yaml new file mode 100644 index 00000000..41329d82 --- /dev/null +++ b/packages/system/fluxcd/charts/flux-instance/templates/healthcheck.yaml @@ -0,0 +1,78 @@ +{{- if .Values.healthcheck.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ .Release.Name }}-healthcheck" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + template: + metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + spec: + restartPolicy: Never + {{- with .Values.healthcheck.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.healthcheck.serviceAccount.name }} + {{- with .Values.healthcheck.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.healthcheck.hostNetwork }} + hostNetwork: true + {{- end }} + containers: + - name: healthcheck + image: "{{ .Values.healthcheck.image.repository }}:{{ .Values.healthcheck.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: "{{ .Values.healthcheck.image.imagePullPolicy }}" + args: + - wait + - instance + - {{ include "flux-instance.fullname" . }} + - --namespace={{ .Release.Namespace }} + - --timeout={{ .Values.healthcheck.timeout }} + {{- range .Values.healthcheck.extraArgs }} + - {{ . }} + {{- end }} + {{- with .Values.healthcheck.envs }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.healthcheck.securityContext | nindent 12 }} + resources: + {{- toYaml .Values.healthcheck.resources | nindent 12 }} + {{- with .Values.healthcheck.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.healthcheck.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.healthcheck.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.healthcheck.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.healthcheck.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml b/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml index 418b21c2..57688811 100644 --- a/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml +++ b/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml @@ -24,7 +24,12 @@ spec: imagePullSecret: {{ .Values.instance.distribution.imagePullSecret }} {{- end }} components: {{ .Values.instance.components | toYaml | nindent 4 }} - cluster: {{ .Values.instance.cluster | toYaml | nindent 4 }} + cluster: + {{- range $key, $value := .Values.instance.cluster }} + {{- if not (and (kindIs "string" $value) (eq $value "")) }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} {{- if or .Values.instance.commonMetadata.annotations .Values.instance.commonMetadata.labels }} commonMetadata: {{- with .Values.instance.commonMetadata.annotations }} diff --git a/packages/system/fluxcd/charts/flux-instance/templates/serviceaccount.yaml b/packages/system/fluxcd/charts/flux-instance/templates/serviceaccount.yaml new file mode 100644 index 00000000..38448f3d --- /dev/null +++ b/packages/system/fluxcd/charts/flux-instance/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.healthcheck.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.healthcheck.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "flux-instance.labels" . | nindent 4 }} + {{- with .Values.commonLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.healthcheck.serviceAccount.automount }} +{{- end }} diff --git a/packages/system/fluxcd/charts/flux-instance/values.schema.json b/packages/system/fluxcd/charts/flux-instance/values.schema.json index cf07b60f..adc8cdda 100644 --- a/packages/system/fluxcd/charts/flux-instance/values.schema.json +++ b/packages/system/fluxcd/charts/flux-instance/values.schema.json @@ -1,20 +1,275 @@ { "$schema": "https://json-schema.org/draft/2019-09/schema", + "type": "object", "properties": { "commonAnnotations": { - "properties": {}, "type": "object" }, "commonLabels": { - "properties": {}, "type": "object" }, "fullnameOverride": { "type": "string" }, + "healthcheck": { + "type": "object", + "required": [ + "resources", + "securityContext" + ], + "properties": { + "affinity": { + "default": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.io/os", + "operator": "In", + "values": [ + "linux" + ] + } + ] + } + ] + } + } + }, + "type": "object", + "properties": { + "nodeAffinity": { + "type": "object", + "properties": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "type": "object", + "properties": { + "nodeSelectorTerms": { + "type": "array", + "items": { + "type": "object", + "properties": { + "matchExpressions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "values": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + } + }, + "enabled": { + "type": "boolean" + }, + "envs": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object" + } + }, + "extraArgs": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "hostNetwork": { + "default": false, + "type": "boolean" + }, + "image": { + "type": "object", + "required": [ + "repository" + ], + "properties": { + "imagePullPolicy": { + "type": "string", + "enum": [ + "IfNotPresent", + "Always", + "Never" + ] + }, + "pullSecrets": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object" + } + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "nodeSelector": { + "type": "object" + }, + "podSecurityContext": { + "default": { + "fsGroup": 1337 + }, + "type": "object" + }, + "resources": { + "type": "object", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + } + }, + "requests": { + "default": { + "cpu": "100m", + "memory": "64Mi" + }, + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + } + } + } + }, + "securityContext": { + "type": "object", + "properties": { + "allowPrivilegeEscalation": { + "default": false, + "type": "boolean" + }, + "capabilities": { + "default": { + "drop": [ + "ALL" + ] + }, + "type": "object", + "properties": { + "drop": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + } + } + }, + "readOnlyRootFilesystem": { + "default": true, + "type": "boolean" + }, + "runAsNonRoot": { + "default": true, + "type": "boolean" + }, + "seccompProfile": { + "default": { + "type": "RuntimeDefault" + }, + "type": "object", + "properties": { + "type": { + "type": "string" + } + } + } + } + }, + "serviceAccount": { + "default": { + "automount": true, + "create": false, + "name": "flux-operator" + }, + "type": "object", + "properties": { + "automount": { + "type": "boolean" + }, + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } + }, + "timeout": { + "default": "5m", + "type": "string" + }, + "tolerations": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object" + } + }, + "volumeMounts": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object" + } + }, + "volumes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object" + } + } + } + }, "instance": { + "type": "object", + "required": [ + "distribution", + "cluster" + ], "properties": { "cluster": { + "type": "object", "properties": { "domain": { "type": "string" @@ -25,37 +280,46 @@ "networkPolicy": { "type": "boolean" }, + "size": { + "type": "string", + "enum": [ + "", + "small", + "medium", + "large" + ] + }, "tenantDefaultServiceAccount": { "type": "string" }, "type": { + "type": "string", "enum": [ "kubernetes", "openshift", "aws", "azure", "gcp" - ], - "type": "string" + ] } - }, - "type": "object" + } }, "commonMetadata": { + "type": "object", "properties": { "annotations": { - "properties": {}, "type": "object" }, "labels": { - "properties": {}, "type": "object" } - }, - "type": "object" + } }, "components": { + "type": "array", + "uniqueItems": true, "items": { + "type": "string", "enum": [ "source-controller", "kustomize-controller", @@ -63,13 +327,15 @@ "notification-controller", "image-reflector-controller", "image-automation-controller" - ], - "type": "string" - }, - "type": "array", - "uniqueItems": true + ] + } }, "distribution": { + "type": "object", + "required": [ + "version", + "registry" + ], "properties": { "artifact": { "type": "string" @@ -86,39 +352,35 @@ "version": { "type": "string" } - }, - "required": [ - "version", - "registry" - ], - "type": "object" + } }, "kustomize": { + "type": "object", "properties": { "patches": { + "type": "array", "items": { "type": "object" - }, - "type": "array" + } } - }, - "type": "object" + } }, "sharding": { + "type": "object", "properties": { "key": { "type": "string" }, "shards": { + "type": "array", "items": { "type": "string" - }, - "type": "array" + } } - }, - "type": "object" + } }, "storage": { + "type": "object", "properties": { "class": { "type": "string" @@ -126,21 +388,21 @@ "size": { "type": "string" } - }, - "type": "object" + } }, "sync": { + "type": "object", "properties": { "interval": { "type": "string" }, "kind": { + "type": "string", "enum": [ "GitRepository", "OCIRepository", "Bucket" - ], - "type": "string" + ] }, "name": { "type": "string" @@ -160,19 +422,12 @@ "url": { "type": "string" } - }, - "type": "object" + } } - }, - "required": [ - "distribution", - "cluster" - ], - "type": "object" + } }, "nameOverride": { "type": "string" } - }, - "type": "object" + } } diff --git a/packages/system/fluxcd/charts/flux-instance/values.yaml b/packages/system/fluxcd/charts/flux-instance/values.yaml index 5b324b04..2a557dbc 100644 --- a/packages/system/fluxcd/charts/flux-instance/values.yaml +++ b/packages/system/fluxcd/charts/flux-instance/values.yaml @@ -20,6 +20,7 @@ instance: # -- Cluster https://fluxcd.control-plane.io/operator/fluxinstance/#cluster-configuration cluster: # @schema required: true type: kubernetes # @schema enum:[kubernetes,openshift,aws,azure,gcp] + size: "" # @schema enum:['',small,medium,large] domain: "cluster.local" networkPolicy: true multitenant: false @@ -35,7 +36,7 @@ instance: # -- Sharding https://fluxcd.control-plane.io/operator/fluxinstance/#sharding-configuration sharding: # @schema required: false key: "sharding.fluxcd.io/key" - shards: [] # @schema item: string + shards: [ ] # @schema item: string # -- Sync https://fluxcd.control-plane.io/operator/fluxinstance/#sync-configuration sync: # @schema required: false interval: 1m @@ -48,10 +49,101 @@ instance: provider: "" kustomize: # @schema required: false # -- Kustomize patches https://fluxcd.control-plane.io/operator/fluxinstance/#kustomize-patches - patches: [] # @schema item: object + patches: [ ] # @schema item: object # -- Common annotations to add to all deployed objects including pods. commonAnnotations: { } # -- Common labels to add to all deployed objects including pods. commonLabels: { } + +# Healthcheck job settings. +healthcheck: + # -- Enable post-install and post-upgrade health checks. + enabled: false + # -- Health check timeout in Go duration format. + timeout: 5m # @schema default: "5m" + + # Container image settings. + # The image tag defaults to the chart appVersion. + # @ignore + image: + repository: ghcr.io/controlplaneio-fluxcd/flux-operator-cli # @schema required: true + tag: "" + pullSecrets: [ ] # @schema item: object ; uniqueItems: true + imagePullPolicy: IfNotPresent # @schema enum:[IfNotPresent, Always, Never] + + # Container resources requests and limits settings. + # @ignore + resources: # @schema required: true + limits: + cpu: 1000m + memory: 1Gi + requests: # @schema default: {"cpu":"100m","memory":"64Mi"} + cpu: 100m + memory: 64Mi + + # Pod service account settings. + # The name of the service account defaults to the release name. + # @ignore + serviceAccount: # @schema default: {"create":false,"automount":true,"name":"flux-operator"} + create: false + automount: true + name: "flux-operator" + + # Pod security context settings. + # @ignore + podSecurityContext: { } # @schema default: {"fsGroup":1337} + + # Container security context settings. + # The default is compliant with the pod security restricted profile. + # @ignore + securityContext: # @schema required: true + runAsNonRoot: true # @schema default: true + readOnlyRootFilesystem: true # @schema default: true + allowPrivilegeEscalation: false # @schema default: false + capabilities: # @schema default: {"drop":["ALL"]} + drop: # @schema item: string ; uniqueItems: true + - "ALL" + seccompProfile: # @schema default: {"type":"RuntimeDefault"} + type: "RuntimeDefault" + + # Pod affinity and anti-affinity settings. + # @ignore + affinity: # @schema default: {"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"kubernetes.io/os","operator":"In","values":["linux"]}]}]}}} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + + # Pod tolerations settings. + # @ignore + tolerations: [ ] # @schema item: object ; uniqueItems: true + + # Pod Node Selector settings. + # @ignore + nodeSelector: { } # @schema type: object + + # If `true`, the container ports (`8080` and `8081`) are exposed on the host network. + # @ignore + hostNetwork: false # @schema default: false + + # Pod extra volumes. + # @ignore + volumes: [ ] # @schema item: object ; uniqueItems: true + + # Container extra volume mounts. + # @ignore + volumeMounts: [ ] # @schema item: object ; uniqueItems: true + + # Container extra environment variables. + # @ignore + envs: [ ] # @schema item: object ; uniqueItems: true + + # Container extra arguments. + # @ignore + extraArgs: [ ] # @schema item: string ; uniqueItems: true From b26dc63b0146eaefd4cc8fb42f08fb530b3112cf Mon Sep 17 00:00:00 2001 From: Nick Volynkin Date: Mon, 4 Aug 2025 21:37:04 +0300 Subject: [PATCH 29/30] [apps] Use new OpenAPI schema and README generator for tenants Signed-off-by: Nick Volynkin --- packages/apps/tenant/Makefile | 2 +- packages/apps/tenant/README.md | 19 +++--- packages/apps/tenant/values.schema.json | 80 +++++++++++++------------ packages/apps/tenant/values.yaml | 14 ++--- 4 files changed, 60 insertions(+), 55 deletions(-) diff --git a/packages/apps/tenant/Makefile b/packages/apps/tenant/Makefile index 6bb0835c..7017c673 100644 --- a/packages/apps/tenant/Makefile +++ b/packages/apps/tenant/Makefile @@ -1,4 +1,4 @@ include ../../../scripts/package.mk generate: - readme-generator-for-helm -v values.yaml -s values.schema.json -r README.md + cozyvalues-gen -v values.yaml -s values.schema.json -r README.md diff --git a/packages/apps/tenant/README.md b/packages/apps/tenant/README.md index cc26c830..5526c146 100644 --- a/packages/apps/tenant/README.md +++ b/packages/apps/tenant/README.md @@ -69,12 +69,13 @@ tenant-u1 ### Common parameters -| Name | Description | Value | -| ---------------- | --------------------------------------------------------------------------------------------------------------------------- | ------- | -| `host` | The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host). | `""` | -| `etcd` | Deploy own Etcd cluster | `false` | -| `monitoring` | Deploy own Monitoring Stack | `false` | -| `ingress` | Deploy own Ingress Controller | `false` | -| `seaweedfs` | Deploy own SeaweedFS | `false` | -| `isolated` | Enforce tenant namespace with network policies | `true` | -| `resourceQuotas` | Define resource quotas for the tenant | `{}` | +| Name | Description | Type | Value | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------- | --------- | ------- | +| `host` | The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host). | `*string` | `""` | +| `etcd` | Deploy own Etcd cluster | `bool` | `false` | +| `monitoring` | Deploy own Monitoring Stack | `bool` | `false` | +| `ingress` | Deploy own Ingress Controller | `bool` | `false` | +| `seaweedfs` | Deploy own SeaweedFS | `bool` | `false` | +| `isolated` | Enforce tenant namespace with network policies, `true` by default | `bool` | `true` | +| `resourceQuotas` | Define resource quotas for the tenant | `string` | `{}` | + diff --git a/packages/apps/tenant/values.schema.json b/packages/apps/tenant/values.schema.json index a0f4f919..6e89ae6d 100644 --- a/packages/apps/tenant/values.schema.json +++ b/packages/apps/tenant/values.schema.json @@ -1,41 +1,45 @@ { - "properties": { - "etcd": { - "default": false, - "description": "Deploy own Etcd cluster", - "type": "boolean" - }, - "host": { - "default": "", - "description": "The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host).", - "type": "string" - }, - "ingress": { - "default": false, - "description": "Deploy own Ingress Controller", - "type": "boolean" - }, - "isolated": { - "default": true, - "description": "Enforce tenant namespace with network policies", - "type": "boolean" - }, - "monitoring": { - "default": false, - "description": "Deploy own Monitoring Stack", - "type": "boolean" - }, - "resourceQuotas": { - "default": {}, - "description": "Define resource quotas for the tenant", - "type": "object" - }, - "seaweedfs": { - "default": false, - "description": "Deploy own SeaweedFS", - "type": "boolean" - } + "title": "Chart Values", + "type": "object", + "properties": { + "etcd": { + "description": "Deploy own Etcd cluster", + "type": "boolean", + "default": false }, - "title": "Chart Values", - "type": "object" + "host": { + "description": "The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host).", + "type": "string" + }, + "ingress": { + "description": "Deploy own Ingress Controller", + "type": "boolean", + "default": false + }, + "isolated": { + "description": "Enforce tenant namespace with network policies, `true` by default", + "type": "boolean", + "default": true + }, + "monitoring": { + "description": "Deploy own Monitoring Stack", + "type": "boolean", + "default": false + }, + "resourceQuotas": { + "description": "Define resource quotas for the tenant", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "x-kubernetes-int-or-string": true + } + }, + "seaweedfs": { + "description": "Deploy own SeaweedFS", + "type": "boolean", + "default": false + } + } } \ No newline at end of file diff --git a/packages/apps/tenant/values.yaml b/packages/apps/tenant/values.yaml index 7c64b278..ae41bf5b 100644 --- a/packages/apps/tenant/values.yaml +++ b/packages/apps/tenant/values.yaml @@ -1,18 +1,18 @@ ## @section Common parameters -## @param host The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host). -## @param etcd Deploy own Etcd cluster -## @param monitoring Deploy own Monitoring Stack -## @param ingress Deploy own Ingress Controller -## @param seaweedfs Deploy own SeaweedFS -## @param isolated Enforce tenant namespace with network policies -## @param resourceQuotas Define resource quotas for the tenant +## @param host {*string} The hostname used to access tenant services (defaults to using the tenant name as a subdomain for it's parent tenant host). +## @param etcd {bool} Deploy own Etcd cluster +## @param monitoring {bool} Deploy own Monitoring Stack +## @param ingress {bool} Deploy own Ingress Controller +## @param seaweedfs {bool} Deploy own SeaweedFS +## @param isolated {bool} Enforce tenant namespace with network policies, `true` by default host: "" etcd: false monitoring: false ingress: false seaweedfs: false isolated: true +## @param resourceQuotas {map[string]quantity} Define resource quotas for the tenant resourceQuotas: {} # resourceQuotas: # cpu: "1" From ff3305f43ce4ab3f228b15666ceb22c0b5ebd59e Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Thu, 7 Aug 2025 13:37:15 +0300 Subject: [PATCH 30/30] [fix] Disable VPA for VPA The earlier PR was erroneously merged without including an amendment to the existing commits, so now this amendment must be included as a separate patch. See #1301 for details. Signed-off-by: Timofei Larkin --- .../templates/helmreleases/vertical-pod-autoscaler.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml index 9bb81ed8..34375d8c 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml @@ -3,8 +3,8 @@ {{- $clusterDomain := (index $cozyConfig.data "cluster-domain") | default "cozy.local" }} {{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }} {{- $targetTenant := index $myNS.metadata.annotations "namespace.cozystack.io/monitoring" }} +vpaForVPA: false vertical-pod-autoscaler: - vpaForVPA: false recommender: extraArgs: container-name-label: container