diff --git a/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json b/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json new file mode 100644 index 00000000..7e19bad5 --- /dev/null +++ b/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json @@ -0,0 +1,5407 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "expr": "ALERTS{app=~\"clickhouse-operator|zookeeper\"}", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "prometheus alerts", + "showIn": 0, + "step": "30s", + "tagKeys": "chi,pod_name,hostname,exported_namespace,namespace", + "tags": [], + "textFormat": "{{alertstate}}", + "titleFormat": "{{alertname}}", + "type": "tags" + } + ] + }, + "description": "Alitinity Clickhouse Operator metrics exported by Monitoring Agent", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 229, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "hidden", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 43, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "auto" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 10, + "x": 0, + "y": 0 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sort(avg by (hostname)(chi_clickhouse_metric_Uptime{chi=~\"$chi\",hostname=~\"$hostname\"})) OR on () vector(0)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "metric": "chi_clickhouse_metric_Uptime", + "range": true, + "refId": "A", + "step": 60 + } + ], + "title": "Uptime (logarithmic)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Clickhouse operator metrics-exporter fails when grab metrics from clickhouse-server\n\nPlease look pods status\n\nkubectl get pods --all-namespaces | grep clickhouse", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 10, + "y": 0 + }, + "id": 47, + "links": [ + { + "targetBlank": true, + "title": "metric_fetch_errors", + "url": "https://github.com/Altinity/clickhouse-operator/search?q=metric_fetch_errors" + } + ], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(chi_clickhouse_metric_fetch_errors{chi=~\"$chi\",hostname=~\"$hostname\",fetch_type=\"system.metrics\"})", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "Failed Pods", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "For example, version 11.22.33 is translated to 11022033", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "hidden", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 3, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "auto" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 7, + "x": 13, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sort_desc(max by (hostname) (chi_clickhouse_metric_VersionInteger{chi=~\"$chi\",hostname=~\"$hostname\"}))", + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "metric": "chi_clickhouse_metric_VersionInteger", + "range": true, + "refId": "A", + "step": 60 + } + ], + "title": "Version", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 0 + }, + "id": 56, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(chi_clickhouse_metric_NumberOfTables{chi=~\"$chi\",hostname=~\"$hostname\"})", + "instant": false, + "legendFormat": "Tables", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(chi_clickhouse_metric_NumberOfDatabases{chi=~\"$chi\",hostname=~\"$hostname\"})", + "hide": false, + "instant": false, + "legendFormat": "Databases", + "range": true, + "refId": "B" + } + ], + "title": "", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#265d1fd9", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "pattern": "(\\d\\d)(?:00(\\d)|0(\\d\\d)|(\\d\\d\\d))0*(.*)", + "result": { + "index": 0, + "text": "$1.$2$3$4.$5" + } + }, + "type": "regex" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 0 + }, + "id": 62, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Version$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(chi_clickhouse_metric_VersionInteger{chi=~\"$chi\",hostname=~\"$hostname\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Version", + "metric": "chi_clickhouse_metric_VersionInteger", + "range": true, + "refId": "A", + "step": 60 + } + ], + "title": "", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "chi-(.*)\\.svc\\.cluster\\.local", + "renamePattern": "$1" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Check Zookeeper connection, Disk Free space and network interconnection between replicas ASAP", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 10, + "y": 2 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Restore After Failures", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-failures" + }, + { + "targetBlank": true, + "title": "Restore After Data Loss", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-complete-data-loss" + } + ], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(chi_clickhouse_metric_ReadonlyReplica{chi=~\"$chi\",hostname=~\"$hostname\"})", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "ReadOnly replicas", + "type": "stat" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show DNS errors and distributed server-server connections failures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 21, + "links": [ + { + "targetBlank": true, + "title": "Manage Distributed tables", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/system#managing-distributed-tables" + }, + { + "targetBlank": true, + "title": "DNSError", + "url": "https://github.com/ClickHouse/ClickHouse/search?q=DNSError" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_NetworkErrors{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "NetworkErrors {{hostname}}", + "metric": "chi_clickhouse_event_NetworkErrors", + "refId": "A", + "step": 120 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DistributedConnectionFailAtAll{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "DistributedConnectionFailAtAll {{hostname}}", + "metric": "chi_clickhouse_event_DistributedConnectionFailAtAll", + "refId": "B", + "step": 120 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DistributedConnectionFailTry{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "DistributedConnectionFailTry {{hostname}}", + "metric": "chi_clickhouse_event_DistributedConnectionFailTry", + "refId": "C", + "step": 120 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DNSError{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "DNSErrors {{hostname}}", + "metric": "chi_clickhouse_event_NetworkErrors", + "refId": "D", + "step": 120 + } + ], + "title": "DNS and Distributed Connection Errors", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show readonly and partial shutdown replicas, zookeeer exceptions, zookeeer sessions, zookeeper init requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 19, + "links": [ + { + "targetBlank": true, + "title": "Recommened Zookeeper Settings", + "url": "https://clickhouse.com/docs/en/operations/tips#zookeeper" + }, + { + "targetBlank": true, + "title": "system.zookeeper", + "url": "https://clickhouse.com/docs/en/operations/system-tables/zookeeper" + }, + { + "targetBlank": true, + "title": "Replication details", + "url": "https://www.slideshare.net/Altinity/introduction-to-the-mysteries-of-clickhouse-replication-by-robert-hodges-and-altinity-engineering-team" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReadonlyReplica{chi=~\"$chi\",hostname=~\"$hostname\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "ReadonlyReplica {{hostname}}", + "metric": "chi_clickhouse_metric_ReadonlyReplica", + "refId": "D", + "step": 120 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ReplicaPartialShutdown{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "ReplicaPartialShutdown {{hostname}}", + "metric": "chi_clickhouse_event_ReplicaPartialShutdown", + "refId": "E", + "step": 120 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperUserExceptions{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "intervalFactor": 2, + "legendFormat": "ZooKeeperUserExceptions {{hostname}}", + "metric": "chi_clickhouse_event_ZooKeeperUserExceptions", + "refId": "B", + "step": 120 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperInit{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "ZooKeeperInit {{hostname}}", + "metric": "chi_clickhouse_event_ZooKeeperInit", + "refId": "A", + "step": 120 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_metric_ZooKeeperSession{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "ZooKeeperSession {{hostname}}", + "metric": "chi_clickhouse_metric_ZooKeeperSession", + "refId": "C", + "step": 120 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperHardwareExceptions{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "ZooKeeperHardwareExceptions {{hostname}}", + "metric": "chi_clickhouse_event_ZooKeeperUserExceptions", + "refId": "F", + "step": 120 + } + ], + "title": "Replication and ZooKeeper Exceptions", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "delayed query\nNumber of INSERT queries that are throttled due to high number of active data parts for partition in a *MergeTree table.\n\ndelayed blocks\nNumber of times the INSERT of a block to a *MergeTree table was throttled due to high number of active data parts for partition. \n\nrejected blocks\nNumber of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.\n\n\nplease look\nparts_to_delay_insert\nparts_to_throw_insert\n\nin system.merge_tree_settings table", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 5, + "links": [ + { + "targetBlank": true, + "title": "system.parts_log", + "url": "https://clickhouse.com/docs/en/operations/system-tables/part_log" + }, + { + "targetBlank": true, + "title": "system.merge_tree_settings", + "url": "https://clickhouse.com/docs/en/operations/system-tables/merge_tree_settings" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_DelayedInserts{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "delayed queries {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DelayedInserts{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "delayed blocks {{hostname}}", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_RejectedInserts{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "rejected blocks {{hostname}}", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_DistributedFilesToInsert{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "pending distributed files {{ hostname }}", + "refId": "D" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_BrokenDistributedFilesToInsert{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "broken distributed files {{ hostname }}", + "refId": "E" + } + ], + "title": "Delayed/Rejected/Pending Inserts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of executing queries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisWidth": 55, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 29, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepBefore", + "lineWidth": 0, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Overall" + }, + "properties": [ + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 11 + }, + "id": 63, + "links": [ + { + "targetBlank": true, + "title": "max_concurent_queries", + "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max_concurrent_queries" + }, + { + "targetBlank": true, + "title": "max_execution_time", + "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max by (hostname) (max_over_time(chi_clickhouse_metric_Query{chi=~\"$chi\",hostname=~\"$hostname\"}[$__interval])-1) OR on () vector(0) > 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{hostname}}", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "sum(chi_clickhouse_metric_Query{chi=~\"$chi\",hostname=~\"$hostname\"}-1) OR on () vector(0)", + "hide": false, + "interval": "", + "intervalFactor": 5, + "legendFormat": "Overall", + "range": true, + "refId": "Overall", + "step": 10 + } + ], + "title": "Queries (running)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of executing select queries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisWidth": 55, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 8, + "links": [ + { + "targetBlank": true, + "title": "max_concurent_queries", + "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max-concurrent-queries" + }, + { + "targetBlank": true, + "title": "max_execution_time", + "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(chi_clickhouse_event_SelectQuery{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])) OR on () vector(0)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Select", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Select Queries (started per sec)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total amount of memory (bytes) allocated in currently executing queries. \n\nNote that some memory allocations may not be accounted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 11 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "max_memory_usage", + "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryTracking{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "refId": "A", + "step": 10 + } + ], + "title": "Memory for Queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of running INSERT queries. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 30, + "links": [ + { + "targetBlank": true, + "title": "max_memory_usage", + "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "irate(chi_clickhouse_event_InsertQuery{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Insert queries {{hostname}}", + "range": true, + "refId": "C" + } + ], + "title": "Insert Queries (running)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of executing insert queries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisWidth": 55, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 58, + "links": [ + { + "targetBlank": true, + "title": "max_concurent_queries", + "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max_concurrent_queries" + }, + { + "targetBlank": true, + "title": "max_execution_time", + "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(chi_clickhouse_event_InsertQuery{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])) OR on () vector(0)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Select", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Insert Queries (started per sec)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "## Tracks rows of inserted data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 32, + "links": [ + { + "targetBlank": true, + "title": "max_memory_usage", + "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_InsertedRows{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Insert rows {{hostname}}", + "refId": "A" + } + ], + "title": "Rows Inserted", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show how intensive data exchange between replicas in parts", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^max.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFA6B0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^check.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^fetch.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B877D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^(data loss|fetch fail|check fail).+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^replicated merge.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#DEB6F2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 25 + }, + "id": 3, + "links": [ + { + "targetBlank": true, + "title": "How replication works", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "irate(chi_clickhouse_event_ReplicatedDataLoss{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "data loss {{hostname}}", + "refId": "A", + "step": 20 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "irate(chi_clickhouse_event_ReplicatedPartChecks{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "check {{hostname}}", + "refId": "B", + "step": 20 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "irate(chi_clickhouse_event_ReplicatedPartChecksFailed{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "check fail {{hostname}}", + "refId": "C", + "step": 20 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "irate(chi_clickhouse_event_ReplicatedPartFetches{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "fetch {{hostname}}", + "refId": "D", + "step": 20 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "irate(chi_clickhouse_event_ReplicatedPartFailedFetches{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "fetch fail {{hostname}}", + "refId": "E", + "step": 20 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "irate(chi_clickhouse_event_ReplicatedPartFetchesOfMerged{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "fetch merged {{hostname}}", + "refId": "F", + "step": 20 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "irate(chi_clickhouse_event_ReplicatedPartMerges{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "replicated merge {{hostname}}", + "refId": "G", + "step": 20 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReplicasSumInsertsInQueue{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "inserts in queue {{hostname}}", + "refId": "H" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReplicasSumMergesInQueue{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "merges in queue {{hostname}}", + "refId": "I" + } + ], + "title": "Replication Queue Jobs", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show seconds when replicated servers can be delayed relative to current time, when you insert directly in *ReplicatedMegreTree table on one server clickhouse need time to replicate new parts of data to another servers in same shard in background", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^absolute.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^relative.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FADE2A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 25 + }, + "id": 59, + "links": [ + { + "targetBlank": true, + "title": "Replication architecture", + "url": "https://clickhouse.com/docs/en/development/architecture#replication" + }, + { + "targetBlank": true, + "title": "ReplicatedMergeTree engine", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication" + }, + { + "targetBlank": true, + "title": "max_replica_delay_for_distributed_queries", + "url": "https://clickhouse.com/docs/en/operations/settings/settings#settings-max_replica_delay_for_distributed_queries" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReplicasMaxAbsoluteDelay{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "absolute {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ReplicasMaxRelativeDelay{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "relative {{hostname}}", + "refId": "B", + "step": 10 + } + ], + "title": "Max Replica Delay", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of requests to ZooKeeper transactions per seconds.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 25 + }, + "id": 34, + "links": [ + { + "targetBlank": true, + "title": "Replication architecture", + "url": "https://clickhouse.com/docs/en/development/architecture#replication" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_ZooKeeperTransactions{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "transactions {{ hostname }}", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_ZooKeeperRequest{chi=~\"$chi\",hostname=~\"$hostname\"}", + "hide": true, + "legendFormat": "{{ hostname }}", + "refId": "A" + } + ], + "title": "Zookeeper Transactions", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show how intensive background merge processes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 32 + }, + "id": 2, + "links": [ + { + "targetBlank": true, + "title": "START/STOP Merges", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/system#stop-merges" + }, + { + "targetBlank": true, + "title": "MegreTree Engine description", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree" + } + ], + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_Merge{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "merges {{hostname}}", + "refId": "A", + "step": 4 + } + ], + "title": "Merges", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show how intensive background merge processes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 32 + }, + "id": 36, + "links": [ + { + "targetBlank": true, + "title": "START/STOP Merges", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/system#stop-merges" + }, + { + "targetBlank": true, + "title": "MegreTree Engine description", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree" + } + ], + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_MergedRows{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "rows {{hostname}}", + "refId": "B", + "step": 4 + } + ], + "title": "Merged Rows", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show how intensive background merge processes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 32 + }, + "id": 49, + "links": [ + { + "targetBlank": true, + "title": "START/STOP Merges", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/system#stop-merges" + }, + { + "targetBlank": true, + "title": "MegreTree Engine description", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/" + } + ], + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_MergedUncompressedBytes{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "bytes {{hostname}}", + "refId": "B", + "step": 4 + } + ], + "title": "Merged Uncompressed Bytes", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 23, + "links": [ + { + "targetBlank": true, + "title": "system.parts", + "url": "https://clickhouse.com/docs/en/operations/system-tables/parts" + }, + { + "targetBlank": true, + "title": "parts_to_delay_insert", + "url": "https://github.com/ClickHouse/ClickHouse/search?q=parts_to_delay_insert" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by(hostname) (chi_clickhouse_table_parts{chi=~\"$chi\",hostname=~\"$hostname\",active=\"1\"})", + "legendFormat": "Parts {{hostname}}", + "refId": "C" + } + ], + "title": "Active Parts", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*detached_by_user.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*broken.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*(clone|ignored).*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Inactive/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 50, + "links": [ + { + "targetBlank": true, + "title": "system.detached_parts", + "url": "https://clickhouse.com/docs/en/operations/system-tables/detached_parts/" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by(hostname,reason) (chi_clickhouse_metric_DetachedParts{chi=~\"$chi\",hostname=~\"$hostname\"})", + "interval": "", + "legendFormat": "{{reason}} {{hostname}} ", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by(hostname) (chi_clickhouse_table_parts{chi=~\"$chi\",hostname=~\"$hostname\",active=\"0\"})", + "hide": true, + "interval": "", + "legendFormat": "Inactive {{hostname}} ", + "refId": "A" + } + ], + "title": "Detached parts", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Each logical partition defined over `PARTITION BY` contains few physical data \"parts\" ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Custom Partitioning Key", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key/" + }, + { + "targetBlank": true, + "title": "system.parts", + "url": "https://clickhouse.com/docs/en/operations/system-tables/parts" + }, + { + "targetBlank": true, + "title": "system.part_log", + "url": "https://clickhouse.com/docs/en/operations/system-tables/part-log" + } + ], + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MaxPartCountForPartition{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "refId": "A", + "step": 10 + } + ], + "title": "Max Part count for Partition", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Memory size allocated for clickhouse-server process\nAvailable for ClickHouse 20.4+\n\nVIRT \nThe total amount of virtual memory used by the task. It includes all code, data and shared libraries plus pages that have been swapped out.\n\nVIRT = SWAP + RES\n\n\nSWAP -- Swapped size (kb)\nThe swapped out portion of a task's total virtual memory image.\n\nRES -- Resident size (kb)\nThe non-swapped physical memory a task has used.\nRES = CODE + USED DATA.\n\nCODE -- Code size (kb)\nThe amount of physical memory devoted to executable code, also known as the 'text resident set' size or TRS\n\nDATA -- Data+Stack size (kb)\nThe amount of physical memory allocated to other than executable code, also known as the 'data resident set' size or DRS.\n\nSHR -- Shared Mem size (kb)\nThe amount of shared memory used by a task. It simply reflects memory that could be potentially shared with other processes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/VIRT.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/DATA.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/CODE.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/RES.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FADE2A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/SHR.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 46 + }, + "id": 46, + "links": [ + { + "targetBlank": true, + "title": "Describe Linux Process Memory types", + "url": "https://elinux.org/Runtime_Memory_Measurement" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryCode{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "CODE {{ hostname }}", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryResident{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "RES {{ hostname }}", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryShared{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "SHR {{ hostname }}", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryDataAndStack{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "DATA {{ hostname }}", + "refId": "D" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryVirtual{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "VIRT {{ hostname }}", + "refId": "E" + } + ], + "title": " clickhouse-server Process Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Memory size allocated for primary keys", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 46 + }, + "id": 45, + "links": [ + { + "targetBlank": true, + "title": "How to choose right primary key", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#selecting-the-primary-key" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryPrimaryKeyBytesAllocated{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "{{ hostname }}", + "refId": "A" + } + ], + "title": "Primary Keys Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Memory size allocated for dictionaries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 46 + }, + "id": 43, + "links": [ + { + "targetBlank": true, + "title": "system.dictionaries", + "url": "https://clickhouse.com/docs/en/operations/system-tables/dictionaries" + }, + { + "targetBlank": true, + "title": "CREATE DICTIONARY", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/create/dictionary" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MemoryDictionaryBytesAllocated{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "{{ hostname }}", + "refId": "A" + } + ], + "title": "Dictionary Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "shows how much space available in the kubernetes pod\n\nbe careful with multiple volumes configuration, kubernetes volume claims and S3 as storage backend", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 53 + }, + "id": 39, + "links": [ + { + "targetBlank": true, + "title": "system.disks", + "url": "https://clickhouse.com/docs/en/operations/system-tables/disks/" + }, + { + "targetBlank": true, + "title": "Multiple Disk Volumes", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-multiple-volumes" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_DiskFreeBytes{chi=~\"$chi\",hostname=~\"$hostname\"} / chi_clickhouse_metric_DiskTotalBytes{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "{{ disk }} {{hostname}}", + "refId": "A" + } + ], + "title": "Disk Space Free", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Bytes" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + }, + { + "id": "custom.width", + "value": 233 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Rows" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.width", + "value": 118 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "database" + }, + "properties": [ + { + "id": "custom.width", + "value": 199 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "table" + }, + "properties": [ + { + "id": "custom.width", + "value": 238 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Parts" + }, + "properties": [ + { + "id": "custom.width", + "value": 101 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "BytePerRow" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 16, + "x": 8, + "y": 53 + }, + "id": 61, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "frameIndex": 2, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Bytes" + } + ] + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum by (database, table) (chi_clickhouse_table_parts_bytes{chi=~\"$chi\",hostname=~\"$hostname\", active=\"1\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Bytes", + "refId": "Bytes" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum by (database, table) (chi_clickhouse_table_parts_rows{chi=~\"$chi\",hostname=~\"$hostname\", active=\"1\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Rows", + "refId": "Rows" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum by (database, table) (chi_clickhouse_table_parts{chi=~\"$chi\",hostname=~\"$hostname\", active=\"1\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Parts", + "refId": "Parts" + } + ], + "title": "Table Stats", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": {}, + "renameByName": { + "Value #Bytes": "Bytes", + "Value #Parts": "Parts", + "Value #Rows": "Rows" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "BytePerRow", + "binary": { + "left": "Bytes", + "operator": "/", + "reducer": "sum", + "right": "Rows" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total data size for all ClickHouse *MergeTree tables\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 60 + }, + "id": 41, + "links": [ + { + "targetBlank": true, + "title": "system.parts", + "url": "https://clickhouse.com/docs/en/operations/system-tables/parts" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_DiskDataBytes{chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "{{ hostname }}", + "refId": "A" + } + ], + "title": "Clickhouse Data size on Disk", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "BackgroundPoolTask\t\n---\nNumber of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping)\n\n\nBackgroundMovePoolTask\n---\nNumber of active tasks in BackgroundProcessingPool for moves\n\n\nBackgroundSchedulePoolTask\t\n---\nA number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 67 + }, + "id": 9, + "links": [ + { + "targetBlank": true, + "title": "FETCH PARTITION", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter/partition#fetch-partitionpart" + }, + { + "targetBlank": true, + "title": "Mutations of data", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter#mutations" + }, + { + "targetBlank": true, + "title": "Data TTL", + "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl" + }, + { + "targetBlank": true, + "title": "MOVE PARTITION", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter/partition#move-partitionpart" + } + ], + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_BackgroundPoolTask{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "merge, mutate, fetch {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_BackgroundSchedulePoolTask{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "clean, alter, replica re-init {{hostname}}", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_BackgroundMovePoolTask{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "moves {{hostname}}", + "refId": "C", + "step": 10 + } + ], + "title": "Background Tasks", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of active mutations (ALTER DELETE/ALTER UPDATE) and parts to mutate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 67 + }, + "id": 26, + "links": [ + { + "targetBlank": true, + "title": "Mutations", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter#mutations" + }, + { + "targetBlank": true, + "title": "system.mutations", + "url": "https://clickhouse.com/docs/en/operations/system-tables/mutations" + }, + { + "targetBlank": true, + "title": "KILL MUTATION", + "url": "https://clickhouse.com/docs/en/sql-reference/statements/kill#kill-mutation" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (hostname) (chi_clickhouse_table_mutations{chi=~\"$chi\",hostname=~\"$hostname\"})", + "legendFormat": "mutations {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (hostname) (chi_clickhouse_table_mutations_parts_to_do{chi=~\"$chi\",hostname=~\"$hostname\"})", + "legendFormat": "parts_to_do {{hostname}}", + "refId": "B" + } + ], + "title": "Mutations", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show which percent of mark files (.mrk) read from memory instead of disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 67 + }, + "id": 11, + "links": [ + { + "targetBlank": true, + "title": "mark_cache_size", + "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#server-mark-cache-size" + }, + { + "targetBlank": true, + "title": "MergeTree architecture", + "url": "https://clickhouse.com/docs/en/development/architecture/#merge-tree" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_MarkCacheHits{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) / (irate(chi_clickhouse_event_MarkCacheHits{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) + irate(chi_clickhouse_event_MarkCacheMisses{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "refId": "A", + "step": 4 + } + ], + "title": "Marks Cache Hit Rate", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "The time which CPU spent on various types of activity ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Disk Read.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Disk Write.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0B400", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Real Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^User Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFF899", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^System Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS IO Wait.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS CPU Wait.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgb(95, 29, 29)", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS CPU Virtual.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B877D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Network Receive.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C0D8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Network Send.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 74 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_DiskReadElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Disk Read syscall {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Disk Write syscall {{hostname}}", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Network Receive {{hostname}}", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Network Send {{hostname}}", + "refId": "D" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_RealTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Real Time {{hostname}}", + "refId": "E" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_UserTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "User Time {{hostname}}", + "refId": "F" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_SystemTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "System Time {{hostname}}", + "refId": "G" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_OSIOWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS IO Wait {{hostname}}", + "refId": "H" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_OSCPUWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Wait {{hostname}}", + "refId": "I" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Virtual {{hostname}}", + "refId": "J" + } + ], + "title": "CPU Time per second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "The time which CPU spent on various types of activity ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Disk Read.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Disk Write.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0B400", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Real Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^User Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFF899", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^System Time.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS IO Wait.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS CPU Wait.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgb(95, 29, 29)", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^OS CPU Virtual.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B877D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Network Receive.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C0D8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Network Send.+/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 74 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_DiskReadElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "legendFormat": "Disk Read syscall {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "legendFormat": "Disk Write syscall {{hostname}}", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "legendFormat": "Network Receive {{hostname}}", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "irate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "legendFormat": "Network Send {{hostname}}", + "refId": "D" + } + ], + "title": "Network / Disk CPU Time per second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 74 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "chi_clickhouse_metric_LoadAverage1{chi=~\"$chi\",hostname=~\"$hostname\"}", + "hide": false, + "interval": "", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "title": "Load Average 1m", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "The time which CPU spent on various types of activity total for the selected period", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 81 + }, + "id": 52, + "interval": "1m", + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DiskReadElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Disk Read syscall {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DiskWriteElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Disk Write syscall {{hostname}}", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Network Receive {{hostname}}", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_NetworkSendElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Network Send {{hostname}}", + "refId": "D" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_RealTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Real Time {{hostname}}", + "refId": "E" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_UserTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "User Time {{hostname}}", + "refId": "F" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_SystemTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "System Time {{hostname}}", + "refId": "G" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_OSIOWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS IO Wait {{hostname}}", + "refId": "H" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_OSCPUWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Wait {{hostname}}", + "refId": "I" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Virtual {{hostname}}", + "refId": "J" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ThrottlerSleepMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Throttler Sleep {{hostname}}", + "refId": "K" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_DelayedInsertsMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "Delayed Insert {{hostname}}", + "refId": "L" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Zookeeper Wait {{hostname}}", + "refId": "M" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_CompileExpressionsMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Compile Expressions {{hostname}}", + "refId": "N" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_MergesTimeMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "Merges {{hostname}}", + "refId": "O" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_RWLockReadersWaitMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "RWLock Reader Wait {{hostname}}", + "refId": "P" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_RWLockWritersWaitMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "RWLock Writer Wait {{hostname}}", + "refId": "Q" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_SelectQueryTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Select Query {{hostname}}", + "refId": "R" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_InsertQueryTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Insert Query {{hostname}}", + "refId": "S" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_S3ReadMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "S3 Read {{hostname}}", + "refId": "T" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(chi_clickhouse_event_S3WriteMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "S3 Write {{hostname}}", + "refId": "U" + } + ], + "title": "CPU Time total", + "type": "piechart" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Show different types of connections for each server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 81 + }, + "id": 48, + "links": [ + { + "targetBlank": true, + "title": "max_connections", + "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max-connections" + }, + { + "targetBlank": true, + "title": "max_distributed_connections", + "url": "https://clickhouse.com/docs/en/operations/settings/settings#max-distributed-connections" + }, + { + "targetBlank": true, + "title": "MySQL Protocol", + "url": "https://clickhouse.com/docs/en/interfaces/mysql/" + }, + { + "targetBlank": true, + "title": "HTTP Protocol", + "url": "https://clickhouse.com/docs/en/interfaces/http/" + }, + { + "targetBlank": true, + "title": "Native Protocol", + "url": "https://clickhouse.com/docs/en/interfaces/tcp/" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_TCPConnection{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "tcp {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_HTTPConnection{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "http {{hostname}}", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_InterserverConnection{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "interserver {{hostname}}", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "chi_clickhouse_metric_MySQLConnection{chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "mysql {{hostname}}", + "refId": "D", + "step": 10 + } + ], + "title": "Connections", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "vm-shortterm", + "value": "24b0d575-0cea-474c-a4eb-c225d250858d" + }, + "includeAll": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, chi)", + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "chi", + "options": [], + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, chi)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",chi=~\"$chi\"}, hostname)", + "includeAll": true, + "label": "Server", + "multi": true, + "name": "hostname", + "options": [], + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",chi=~\"$chi\"}, hostname)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "Altinity ClickHouse Operator Dashboard", + "uid": "clickhouse-operator", + "version": 4, + "weekStart": "" +} diff --git a/packages/extra/monitoring/dashboards.list b/packages/extra/monitoring/dashboards.list index e84a8caa..e0247db7 100644 --- a/packages/extra/monitoring/dashboards.list +++ b/packages/extra/monitoring/dashboards.list @@ -36,3 +36,4 @@ flux/flux-control-plane flux/flux-stats kafka/strimzi-kafka goldpinger/goldpinger +clickhouse/altinity-clickhouse-operator-dashboard diff --git a/packages/system/clickhouse-operator/values.yaml b/packages/system/clickhouse-operator/values.yaml index 287874fc..fc740432 100644 --- a/packages/system/clickhouse-operator/values.yaml +++ b/packages/system/clickhouse-operator/values.yaml @@ -1,4 +1,6 @@ altinity-clickhouse-operator: + serviceMonitor: + enabled: true configs: files: config.yaml: