{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "datasource",
          "uid": "grafana"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "enable": true,
        "expr": "ALERTS{app=~\"clickhouse-operator|zookeeper\"}",
        "hide": false,
        "iconColor": "rgba(255, 96, 96, 1)",
        "limit": 100,
        "name": "prometheus alerts",
        "showIn": 0,
        "step": "30s",
        "tagKeys": "chi,pod_name,hostname,exported_namespace,namespace",
        "tags": [],
        "textFormat": "{{alertstate}}",
        "titleFormat": "{{alertname}}",
        "type": "tags"
      }
    ]
  },
  "description": "Alitinity Clickhouse Operator metrics exported by Monitoring Agent",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": 229,
  "links": [],
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "hidden",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 22,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "log": 2,
              "type": "log"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 43,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "rgba(245, 54, 54, 0.9)",
                "value": null
              }
            ]
          },
          "unit": "s"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "auto"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 4,
        "w": 10,
        "x": 0,
        "y": 0
      },
      "id": 15,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sort(avg by (hostname)(chi_clickhouse_metric_Uptime{chi=~\"$chi\",hostname=~\"$hostname\"})) OR on () vector(0)",
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "{{hostname}}",
          "metric": "chi_clickhouse_metric_Uptime",
          "range": true,
          "refId": "A",
          "step": 60
        }
      ],
      "title": "Uptime (logarithmic)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Clickhouse operator metrics-exporter fails when grab metrics from clickhouse-server\n\nPlease look pods status\n\nkubectl get pods --all-namespaces | grep clickhouse",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 2,
        "w": 3,
        "x": 10,
        "y": 0
      },
      "id": 47,
      "links": [
        {
          "targetBlank": true,
          "title": "metric_fetch_errors",
          "url": "https://github.com/Altinity/clickhouse-operator/search?q=metric_fetch_errors"
        }
      ],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum(chi_clickhouse_metric_fetch_errors{chi=~\"$chi\",hostname=~\"$hostname\",fetch_type=\"system.metrics\"})",
          "intervalFactor": 2,
          "legendFormat": "",
          "refId": "A",
          "step": 60
        }
      ],
      "title": "Failed Pods",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "For example, version 11.22.33 is translated to 11022033",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "hidden",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 3,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "locale"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byType",
              "options": "time"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "auto"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 4,
        "w": 7,
        "x": 13,
        "y": 0
      },
      "id": 17,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sort_desc(max by (hostname) (chi_clickhouse_metric_VersionInteger{chi=~\"$chi\",hostname=~\"$hostname\"}))",
          "intervalFactor": 2,
          "legendFormat": "{{hostname}}",
          "metric": "chi_clickhouse_metric_VersionInteger",
          "range": true,
          "refId": "A",
          "step": 60
        }
      ],
      "title": "Version",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 2,
        "x": 20,
        "y": 0
      },
      "id": 56,
      "options": {
        "colorMode": "none",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "horizontal",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "value_and_name",
        "wideLayout": true
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "max(chi_clickhouse_metric_NumberOfTables{chi=~\"$chi\",hostname=~\"$hostname\"})",
          "instant": false,
          "legendFormat": "Tables",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "max(chi_clickhouse_metric_NumberOfDatabases{chi=~\"$chi\",hostname=~\"$hostname\"})",
          "hide": false,
          "instant": false,
          "legendFormat": "Databases",
          "range": true,
          "refId": "B"
        }
      ],
      "title": "",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "#265d1fd9",
                "value": null
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Value"
            },
            "properties": [
              {
                "id": "mappings",
                "value": [
                  {
                    "options": {
                      "pattern": "(\\d\\d)(?:00(\\d)|0(\\d\\d)|(\\d\\d\\d))0*(.*)",
                      "result": {
                        "index": 0,
                        "text": "$1.$2$3$4.$5"
                      }
                    },
                    "type": "regex"
                  }
                ]
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 4,
        "w": 2,
        "x": 22,
        "y": 0
      },
      "id": 62,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "/^Version$/",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "value_and_name",
        "wideLayout": true
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "editorMode": "code",
          "exemplar": true,
          "expr": "max(chi_clickhouse_metric_VersionInteger{chi=~\"$chi\",hostname=~\"$hostname\"})",
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "Version",
          "metric": "chi_clickhouse_metric_VersionInteger",
          "range": true,
          "refId": "A",
          "step": 60
        }
      ],
      "title": "",
      "transformations": [
        {
          "id": "renameByRegex",
          "options": {
            "regex": "chi-(.*)\\.svc\\.cluster\\.local",
            "renamePattern": "$1"
          }
        },
        {
          "id": "convertFieldType",
          "options": {
            "conversions": [
              {
                "destinationType": "string",
                "targetField": "Value"
              }
            ],
            "fields": {}
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Check Zookeeper connection, Disk Free space and network interconnection between replicas ASAP",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 2,
        "w": 3,
        "x": 10,
        "y": 2
      },
      "id": 6,
      "links": [
        {
          "targetBlank": true,
          "title": "Restore After Failures",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-failures"
        },
        {
          "targetBlank": true,
          "title": "Restore After Data Loss",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-complete-data-loss"
        }
      ],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum(chi_clickhouse_metric_ReadonlyReplica{chi=~\"$chi\",hostname=~\"$hostname\"})",
          "intervalFactor": 2,
          "legendFormat": "",
          "refId": "A",
          "step": 60
        }
      ],
      "title": "ReadOnly replicas",
      "type": "stat"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Show DNS errors and distributed server-server connections failures",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "stepAfter",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 4
      },
      "id": 21,
      "links": [
        {
          "targetBlank": true,
          "title": "Manage Distributed tables",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/system#managing-distributed-tables"
        },
        {
          "targetBlank": true,
          "title": "DNSError",
          "url": "https://github.com/ClickHouse/ClickHouse/search?q=DNSError"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_NetworkErrors{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "NetworkErrors {{hostname}}",
          "metric": "chi_clickhouse_event_NetworkErrors",
          "refId": "A",
          "step": 120
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_DistributedConnectionFailAtAll{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "DistributedConnectionFailAtAll {{hostname}}",
          "metric": "chi_clickhouse_event_DistributedConnectionFailAtAll",
          "refId": "B",
          "step": 120
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_DistributedConnectionFailTry{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "DistributedConnectionFailTry {{hostname}}",
          "metric": "chi_clickhouse_event_DistributedConnectionFailTry",
          "refId": "C",
          "step": 120
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_DNSError{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "DNSErrors {{hostname}}",
          "metric": "chi_clickhouse_event_NetworkErrors",
          "refId": "D",
          "step": 120
        }
      ],
      "title": "DNS and Distributed Connection Errors",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Show readonly and partial shutdown replicas, zookeeer exceptions, zookeeer sessions, zookeeper init requests",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "stepAfter",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 4
      },
      "id": 19,
      "links": [
        {
          "targetBlank": true,
          "title": "Recommened Zookeeper Settings",
          "url": "https://clickhouse.com/docs/en/operations/tips#zookeeper"
        },
        {
          "targetBlank": true,
          "title": "system.zookeeper",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/zookeeper"
        },
        {
          "targetBlank": true,
          "title": "Replication details",
          "url": "https://www.slideshare.net/Altinity/introduction-to-the-mysteries-of-clickhouse-replication-by-robert-hodges-and-altinity-engineering-team"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_ReadonlyReplica{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "hide": false,
          "intervalFactor": 2,
          "legendFormat": "ReadonlyReplica {{hostname}}",
          "metric": "chi_clickhouse_metric_ReadonlyReplica",
          "refId": "D",
          "step": 120
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_ReplicaPartialShutdown{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "ReplicaPartialShutdown {{hostname}}",
          "metric": "chi_clickhouse_event_ReplicaPartialShutdown",
          "refId": "E",
          "step": 120
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_ZooKeeperUserExceptions{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "ZooKeeperUserExceptions  {{hostname}}",
          "metric": "chi_clickhouse_event_ZooKeeperUserExceptions",
          "refId": "B",
          "step": 120
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_ZooKeeperInit{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "ZooKeeperInit {{hostname}}",
          "metric": "chi_clickhouse_event_ZooKeeperInit",
          "refId": "A",
          "step": 120
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_metric_ZooKeeperSession{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "ZooKeeperSession  {{hostname}}",
          "metric": "chi_clickhouse_metric_ZooKeeperSession",
          "refId": "C",
          "step": 120
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_ZooKeeperHardwareExceptions{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "ZooKeeperHardwareExceptions  {{hostname}}",
          "metric": "chi_clickhouse_event_ZooKeeperUserExceptions",
          "refId": "F",
          "step": 120
        }
      ],
      "title": "Replication and ZooKeeper Exceptions",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "delayed query\nNumber of INSERT queries that are throttled due to high number of active data parts for partition in a *MergeTree table.\n\ndelayed blocks\nNumber of times the INSERT of a block to a *MergeTree table was throttled due to high number of active data parts for partition. \n\nrejected blocks\nNumber of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.\n\n\nplease look\nparts_to_delay_insert\nparts_to_throw_insert\n\nin system.merge_tree_settings table",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 4
      },
      "id": 5,
      "links": [
        {
          "targetBlank": true,
          "title": "system.parts_log",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/part_log"
        },
        {
          "targetBlank": true,
          "title": "system.merge_tree_settings",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/merge_tree_settings"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_DelayedInserts{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "delayed  queries {{hostname}}",
          "refId": "A",
          "step": 10
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_DelayedInserts{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "delayed blocks {{hostname}}",
          "refId": "B",
          "step": 10
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_RejectedInserts{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "rejected blocks {{hostname}}",
          "refId": "C",
          "step": 10
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_DistributedFilesToInsert{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "pending distributed files {{ hostname }}",
          "refId": "D"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_BrokenDistributedFilesToInsert{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "broken distributed files {{ hostname }}",
          "refId": "E"
        }
      ],
      "title": "Delayed/Rejected/Pending Inserts",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Number of executing queries",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "axisWidth": 55,
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 29,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "stepBefore",
            "lineWidth": 0,
            "pointSize": 2,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Overall"
            },
            "properties": [
              {
                "id": "custom.lineInterpolation",
                "value": "smooth"
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "custom.lineStyle",
                "value": {
                  "fill": "solid"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 1
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 11
      },
      "id": 63,
      "links": [
        {
          "targetBlank": true,
          "title": "max_concurent_queries",
          "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max_concurrent_queries"
        },
        {
          "targetBlank": true,
          "title": "max_execution_time",
          "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "max",
            "sum"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false,
          "sortBy": "Max",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "editorMode": "code",
          "exemplar": true,
          "expr": "max by (hostname) (max_over_time(chi_clickhouse_metric_Query{chi=~\"$chi\",hostname=~\"$hostname\"}[$__interval])-1) OR on () vector(0) > 0",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{hostname}}",
          "range": true,
          "refId": "A",
          "step": 10
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "editorMode": "code",
          "exemplar": true,
          "expr": "sum(chi_clickhouse_metric_Query{chi=~\"$chi\",hostname=~\"$hostname\"}-1) OR on () vector(0)",
          "hide": false,
          "interval": "",
          "intervalFactor": 5,
          "legendFormat": "Overall",
          "range": true,
          "refId": "Overall",
          "step": 10
        }
      ],
      "title": "Queries (running)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Number of executing select queries",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "axisWidth": 55,
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 25,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 11
      },
      "id": 8,
      "links": [
        {
          "targetBlank": true,
          "title": "max_concurent_queries",
          "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max-concurrent-queries"
        },
        {
          "targetBlank": true,
          "title": "max_execution_time",
          "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": "sum(rate(chi_clickhouse_event_SelectQuery{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])) OR on () vector(0)",
          "hide": false,
          "intervalFactor": 2,
          "legendFormat": "Select",
          "range": true,
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Select Queries (started per sec)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Total amount of memory (bytes) allocated in currently executing queries. \n\nNote that some memory allocations may not be accounted.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 11
      },
      "id": 13,
      "links": [
        {
          "targetBlank": true,
          "title": "max_memory_usage",
          "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MemoryTracking{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "{{hostname}}",
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Memory for Queries",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Number of running INSERT queries. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 18
      },
      "id": 30,
      "links": [
        {
          "targetBlank": true,
          "title": "max_memory_usage",
          "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "irate(chi_clickhouse_event_InsertQuery{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Insert queries {{hostname}}",
          "range": true,
          "refId": "C"
        }
      ],
      "title": "Insert Queries (running)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Number of executing insert queries",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "axisWidth": 55,
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 25,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 18
      },
      "id": 58,
      "links": [
        {
          "targetBlank": true,
          "title": "max_concurent_queries",
          "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max_concurrent_queries"
        },
        {
          "targetBlank": true,
          "title": "max_execution_time",
          "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": "sum(rate(chi_clickhouse_event_InsertQuery{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])) OR on () vector(0)",
          "hide": false,
          "intervalFactor": 2,
          "legendFormat": "Select",
          "range": true,
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Insert Queries (started per sec)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "## Tracks rows of inserted data.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 18
      },
      "id": 32,
      "links": [
        {
          "targetBlank": true,
          "title": "max_memory_usage",
          "url": "https://clickhouse.com/docs/en/operations/settings/query-complexity#settings_max_memory_usage"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.4.0",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_InsertedRows{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Insert rows {{hostname}}",
          "refId": "A"
        }
      ],
      "title": "Rows Inserted",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Show how intensive data exchange between replicas in parts",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^max.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FFA6B0",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^check.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FF9830",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^fetch.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#B877D9",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^(data loss|fetch fail|check fail).+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#C4162A",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^replicated merge.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#DEB6F2",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 25
      },
      "id": 3,
      "links": [
        {
          "targetBlank": true,
          "title": "How replication works",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "irate(chi_clickhouse_event_ReplicatedDataLoss{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "data loss {{hostname}}",
          "refId": "A",
          "step": 20
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "irate(chi_clickhouse_event_ReplicatedPartChecks{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "check {{hostname}}",
          "refId": "B",
          "step": 20
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "irate(chi_clickhouse_event_ReplicatedPartChecksFailed{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "check fail {{hostname}}",
          "refId": "C",
          "step": 20
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "irate(chi_clickhouse_event_ReplicatedPartFetches{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "fetch {{hostname}}",
          "refId": "D",
          "step": 20
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "irate(chi_clickhouse_event_ReplicatedPartFailedFetches{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "fetch fail {{hostname}}",
          "refId": "E",
          "step": 20
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "irate(chi_clickhouse_event_ReplicatedPartFetchesOfMerged{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "fetch merged {{hostname}}",
          "refId": "F",
          "step": 20
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "irate(chi_clickhouse_event_ReplicatedPartMerges{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "replicated merge {{hostname}}",
          "refId": "G",
          "step": 20
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_ReplicasSumInsertsInQueue{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "inserts in queue {{hostname}}",
          "refId": "H"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_ReplicasSumMergesInQueue{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "merges in queue {{hostname}}",
          "refId": "I"
        }
      ],
      "title": "Replication Queue Jobs",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Show seconds when replicated servers can be delayed relative to current time,  when you insert directly in *ReplicatedMegreTree table on one server clickhouse need time to replicate new parts of data to another servers in same shard in background",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^absolute.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#F2495C",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^relative.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FADE2A",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 25
      },
      "id": 59,
      "links": [
        {
          "targetBlank": true,
          "title": "Replication architecture",
          "url": "https://clickhouse.com/docs/en/development/architecture#replication"
        },
        {
          "targetBlank": true,
          "title": "ReplicatedMergeTree engine",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication"
        },
        {
          "targetBlank": true,
          "title": "max_replica_delay_for_distributed_queries",
          "url": "https://clickhouse.com/docs/en/operations/settings/settings#settings-max_replica_delay_for_distributed_queries"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_ReplicasMaxAbsoluteDelay{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "absolute {{hostname}}",
          "refId": "A",
          "step": 10
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_ReplicasMaxRelativeDelay{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "relative {{hostname}}",
          "refId": "B",
          "step": 10
        }
      ],
      "title": "Max Replica Delay",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Number of requests to ZooKeeper transactions per seconds.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 25
      },
      "id": 34,
      "links": [
        {
          "targetBlank": true,
          "title": "Replication architecture",
          "url": "https://clickhouse.com/docs/en/development/architecture#replication"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_ZooKeeperTransactions{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "transactions {{ hostname }}",
          "refId": "B"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_ZooKeeperRequest{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "hide": true,
          "legendFormat": "{{ hostname }}",
          "refId": "A"
        }
      ],
      "title": "Zookeeper Transactions",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Show how intensive background merge processes",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 32
      },
      "id": 2,
      "links": [
        {
          "targetBlank": true,
          "title": "START/STOP Merges",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/system#stop-merges"
        },
        {
          "targetBlank": true,
          "title": "MegreTree Engine description",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_Merge{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "merges {{hostname}}",
          "refId": "A",
          "step": 4
        }
      ],
      "title": "Merges",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Show how intensive background merge processes",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 32
      },
      "id": 36,
      "links": [
        {
          "targetBlank": true,
          "title": "START/STOP Merges",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/system#stop-merges"
        },
        {
          "targetBlank": true,
          "title": "MegreTree Engine description",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_MergedRows{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "rows {{hostname}}",
          "refId": "B",
          "step": 4
        }
      ],
      "title": "Merged Rows",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Show how intensive background merge processes",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "decbytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 32
      },
      "id": 49,
      "links": [
        {
          "targetBlank": true,
          "title": "START/STOP Merges",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/system#stop-merges"
        },
        {
          "targetBlank": true,
          "title": "MegreTree Engine description",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_MergedUncompressedBytes{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "intervalFactor": 2,
          "legendFormat": "bytes {{hostname}}",
          "refId": "B",
          "step": 4
        }
      ],
      "title": "Merged Uncompressed Bytes",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 39
      },
      "id": 23,
      "links": [
        {
          "targetBlank": true,
          "title": "system.parts",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/parts"
        },
        {
          "targetBlank": true,
          "title": "parts_to_delay_insert",
          "url": "https://github.com/ClickHouse/ClickHouse/search?q=parts_to_delay_insert"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by(hostname) (chi_clickhouse_table_parts{chi=~\"$chi\",hostname=~\"$hostname\",active=\"1\"})",
          "legendFormat": "Parts {{hostname}}",
          "refId": "C"
        }
      ],
      "title": "Active Parts",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/.*detached_by_user.*/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#CA95E5",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/.*broken.*/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#E02F44",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/.*(clone|ignored).*/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FFEE52",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Inactive/"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "hidden"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 39
      },
      "id": 50,
      "links": [
        {
          "targetBlank": true,
          "title": "system.detached_parts",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/detached_parts/"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by(hostname,reason) (chi_clickhouse_metric_DetachedParts{chi=~\"$chi\",hostname=~\"$hostname\"})",
          "interval": "",
          "legendFormat": "{{reason}} {{hostname}} ",
          "refId": "C"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by(hostname) (chi_clickhouse_table_parts{chi=~\"$chi\",hostname=~\"$hostname\",active=\"0\"})",
          "hide": true,
          "interval": "",
          "legendFormat": "Inactive {{hostname}} ",
          "refId": "A"
        }
      ],
      "title": "Detached parts",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Each logical partition defined over `PARTITION BY` contains few physical data \"parts\" ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 39
      },
      "id": 4,
      "links": [
        {
          "targetBlank": true,
          "title": "Custom Partitioning Key",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key/"
        },
        {
          "targetBlank": true,
          "title": "system.parts",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/parts"
        },
        {
          "targetBlank": true,
          "title": "system.part_log",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/part-log"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MaxPartCountForPartition{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "{{hostname}}",
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Max Part count for Partition",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Memory size allocated for clickhouse-server process\nAvailable for ClickHouse 20.4+\n\nVIRT \nThe total amount of virtual memory used by the task. It includes all code, data and shared libraries plus pages that have been swapped out.\n\nVIRT = SWAP + RES\n\n\nSWAP -- Swapped size (kb)\nThe swapped out portion of a task's total virtual memory image.\n\nRES -- Resident size (kb)\nThe non-swapped physical memory a task has used.\nRES = CODE + USED DATA.\n\nCODE -- Code size (kb)\nThe amount of physical memory devoted to executable code, also known as the 'text resident set' size or TRS\n\nDATA -- Data+Stack size (kb)\nThe amount of physical memory allocated to other than executable code, also known as the 'data resident set' size or DRS.\n\nSHR -- Shared Mem size (kb)\nThe amount of shared memory used by a task. It simply reflects memory that could be potentially shared with other processes.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "decbytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/VIRT.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#73BF69",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/DATA.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#C4162A",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/CODE.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FF9830",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/RES.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FADE2A",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/SHR.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#5794F2",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 46
      },
      "id": 46,
      "links": [
        {
          "targetBlank": true,
          "title": "Describe Linux Process Memory types",
          "url": "https://elinux.org/Runtime_Memory_Measurement"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MemoryCode{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "CODE {{ hostname }}",
          "refId": "A"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MemoryResident{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "RES {{ hostname }}",
          "refId": "B"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MemoryShared{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "SHR {{ hostname }}",
          "refId": "C"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MemoryDataAndStack{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "DATA {{ hostname }}",
          "refId": "D"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MemoryVirtual{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "VIRT {{ hostname }}",
          "refId": "E"
        }
      ],
      "title": " clickhouse-server Process Memory",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Memory size allocated for primary keys",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 46
      },
      "id": 45,
      "links": [
        {
          "targetBlank": true,
          "title": "How to choose right primary key",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#selecting-the-primary-key"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MemoryPrimaryKeyBytesAllocated{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "{{ hostname }}",
          "refId": "A"
        }
      ],
      "title": "Primary Keys Memory",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Memory size allocated for dictionaries",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "decbytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 46
      },
      "id": 43,
      "links": [
        {
          "targetBlank": true,
          "title": "system.dictionaries",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/dictionaries"
        },
        {
          "targetBlank": true,
          "title": "CREATE DICTIONARY",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/create/dictionary"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MemoryDictionaryBytesAllocated{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "{{ hostname }}",
          "refId": "A"
        }
      ],
      "title": "Dictionary Memory",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "shows how much space available in the kubernetes pod\n\nbe careful with multiple volumes configuration, kubernetes volume claims and S3 as storage backend",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 53
      },
      "id": 39,
      "links": [
        {
          "targetBlank": true,
          "title": "system.disks",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/disks/"
        },
        {
          "targetBlank": true,
          "title": "Multiple Disk Volumes",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-multiple-volumes"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_DiskFreeBytes{chi=~\"$chi\",hostname=~\"$hostname\"} / chi_clickhouse_metric_DiskTotalBytes{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "{{ disk }} {{hostname}}",
          "refId": "A"
        }
      ],
      "title": "Disk Space Free",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "type": "auto"
            },
            "filterable": true,
            "inspect": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Bytes"
            },
            "properties": [
              {
                "id": "unit",
                "value": "decbytes"
              },
              {
                "id": "custom.cellOptions",
                "value": {
                  "mode": "gradient",
                  "type": "gauge"
                }
              },
              {
                "id": "color",
                "value": {
                  "mode": "continuous-BlPu"
                }
              },
              {
                "id": "custom.width",
                "value": 233
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Rows"
            },
            "properties": [
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "custom.width",
                "value": 118
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "database"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 199
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "table"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 238
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Parts"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 101
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "BytePerRow"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 120
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 14,
        "w": 16,
        "x": 8,
        "y": 53
      },
      "id": 61,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": true
        },
        "frameIndex": 2,
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "Bytes"
          }
        ]
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "editorMode": "code",
          "exemplar": false,
          "expr": "sum by (database, table) (chi_clickhouse_table_parts_bytes{chi=~\"$chi\",hostname=~\"$hostname\", active=\"1\"})",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "Bytes",
          "refId": "Bytes"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "editorMode": "code",
          "exemplar": false,
          "expr": "sum by (database, table) (chi_clickhouse_table_parts_rows{chi=~\"$chi\",hostname=~\"$hostname\", active=\"1\"})",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "Rows",
          "refId": "Rows"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "editorMode": "code",
          "exemplar": false,
          "expr": "sum by (database, table) (chi_clickhouse_table_parts{chi=~\"$chi\",hostname=~\"$hostname\", active=\"1\"})",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "Parts",
          "refId": "Parts"
        }
      ],
      "title": "Table Stats",
      "transformations": [
        {
          "id": "merge",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true
            },
            "indexByName": {},
            "renameByName": {
              "Value #Bytes": "Bytes",
              "Value #Parts": "Parts",
              "Value #Rows": "Rows"
            }
          }
        },
        {
          "id": "calculateField",
          "options": {
            "alias": "BytePerRow",
            "binary": {
              "left": "Bytes",
              "operator": "/",
              "reducer": "sum",
              "right": "Rows"
            },
            "mode": "binary",
            "reduce": {
              "reducer": "sum"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Total data size for all ClickHouse *MergeTree tables\n\n",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "decbytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 60
      },
      "id": 41,
      "links": [
        {
          "targetBlank": true,
          "title": "system.parts",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/parts"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_DiskDataBytes{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "legendFormat": "{{ hostname }}",
          "refId": "A"
        }
      ],
      "title": "Clickhouse Data size on Disk",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "BackgroundPoolTask\t\n---\nNumber of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping)\n\n\nBackgroundMovePoolTask\n---\nNumber of active tasks in BackgroundProcessingPool for moves\n\n\nBackgroundSchedulePoolTask\t\n---\nA number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 67
      },
      "id": 9,
      "links": [
        {
          "targetBlank": true,
          "title": "FETCH PARTITION",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter/partition#fetch-partitionpart"
        },
        {
          "targetBlank": true,
          "title": "Mutations of data",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter#mutations"
        },
        {
          "targetBlank": true,
          "title": "Data TTL",
          "url": "https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl"
        },
        {
          "targetBlank": true,
          "title": "MOVE PARTITION",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter/partition#move-partitionpart"
        }
      ],
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_BackgroundPoolTask{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "merge, mutate, fetch {{hostname}}",
          "refId": "A",
          "step": 10
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_BackgroundSchedulePoolTask{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "clean, alter, replica re-init {{hostname}}",
          "refId": "B",
          "step": 10
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_BackgroundMovePoolTask{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "moves {{hostname}}",
          "refId": "C",
          "step": 10
        }
      ],
      "title": "Background Tasks",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Number of active mutations (ALTER DELETE/ALTER UPDATE) and parts to mutate",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 67
      },
      "id": 26,
      "links": [
        {
          "targetBlank": true,
          "title": "Mutations",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/alter#mutations"
        },
        {
          "targetBlank": true,
          "title": "system.mutations",
          "url": "https://clickhouse.com/docs/en/operations/system-tables/mutations"
        },
        {
          "targetBlank": true,
          "title": "KILL MUTATION",
          "url": "https://clickhouse.com/docs/en/sql-reference/statements/kill#kill-mutation"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (hostname) (chi_clickhouse_table_mutations{chi=~\"$chi\",hostname=~\"$hostname\"})",
          "legendFormat": "mutations {{hostname}}",
          "refId": "A"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (hostname) (chi_clickhouse_table_mutations_parts_to_do{chi=~\"$chi\",hostname=~\"$hostname\"})",
          "legendFormat": "parts_to_do {{hostname}}",
          "refId": "B"
        }
      ],
      "title": "Mutations",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Show which percent of mark files (.mrk) read from memory instead of disk",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 67
      },
      "id": 11,
      "links": [
        {
          "targetBlank": true,
          "title": "mark_cache_size",
          "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#server-mark-cache-size"
        },
        {
          "targetBlank": true,
          "title": "MergeTree architecture",
          "url": "https://clickhouse.com/docs/en/development/architecture/#merge-tree"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_MarkCacheHits{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) / (irate(chi_clickhouse_event_MarkCacheHits{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) + irate(chi_clickhouse_event_MarkCacheMisses{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]))",
          "hide": false,
          "intervalFactor": 2,
          "legendFormat": "{{hostname}}",
          "refId": "A",
          "step": 4
        }
      ],
      "title": "Marks Cache Hit Rate",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "The time which CPU spent on various types of activity ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "µs"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Disk Read.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FF9830",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Disk Write.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#E0B400",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Real Time.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#73BF69",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^User Time.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FFF899",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^System Time.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#F2495C",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^OS IO Wait.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#C4162A",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^OS CPU Wait.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "rgb(95, 29, 29)",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^OS CPU Virtual.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#B877D9",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Network Receive.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#C0D8FF",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Network Send.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#8AB8FF",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 74
      },
      "id": 51,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_DiskReadElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": true,
          "legendFormat": "Disk Read syscall {{hostname}}",
          "refId": "A"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": true,
          "legendFormat": "Disk Write syscall {{hostname}}",
          "refId": "B"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": true,
          "legendFormat": "Network Receive {{hostname}}",
          "refId": "C"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": true,
          "legendFormat": "Network Send {{hostname}}",
          "refId": "D"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_RealTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Real Time {{hostname}}",
          "refId": "E"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_UserTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "User Time {{hostname}}",
          "refId": "F"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_SystemTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "System Time {{hostname}}",
          "refId": "G"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_OSIOWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "OS IO Wait {{hostname}}",
          "refId": "H"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_OSCPUWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "OS CPU Wait {{hostname}}",
          "refId": "I"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "OS CPU Virtual {{hostname}}",
          "refId": "J"
        }
      ],
      "title": "CPU Time per second",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "The time which CPU spent on various types of activity ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "µs"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Disk Read.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FF9830",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Disk Write.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#E0B400",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Real Time.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#73BF69",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^User Time.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#FFF899",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^System Time.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#F2495C",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^OS IO Wait.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#C4162A",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^OS CPU Wait.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "rgb(95, 29, 29)",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^OS CPU Virtual.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#B877D9",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Network Receive.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#C0D8FF",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/^Network Send.+/"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#8AB8FF",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 74
      },
      "id": 54,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_DiskReadElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "legendFormat": "Disk Read syscall {{hostname}}",
          "refId": "A"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "legendFormat": "Disk Write syscall {{hostname}}",
          "refId": "B"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "legendFormat": "Network Receive {{hostname}}",
          "refId": "C"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "irate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "hide": false,
          "legendFormat": "Network Send {{hostname}}",
          "refId": "D"
        }
      ],
      "title": "Network / Disk CPU Time per second",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 74
      },
      "id": 55,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "chi_clickhouse_metric_LoadAverage1{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "hide": false,
          "interval": "",
          "legendFormat": "{{hostname}}",
          "refId": "A"
        }
      ],
      "title": "Load Average 1m",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "The time which CPU spent on various types of activity total for the selected period",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            }
          },
          "mappings": [],
          "unit": "µs"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 16,
        "x": 0,
        "y": 81
      },
      "id": 52,
      "interval": "1m",
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "right",
          "showLegend": true
        },
        "pieType": "pie",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_DiskReadElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Disk Read syscall {{hostname}}",
          "refId": "A"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_DiskWriteElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Disk Write syscall {{hostname}}",
          "refId": "B"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Network Receive {{hostname}}",
          "refId": "C"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_NetworkSendElapsedMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Network Send {{hostname}}",
          "refId": "D"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_RealTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Real Time {{hostname}}",
          "refId": "E"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_UserTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "User Time {{hostname}}",
          "refId": "F"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_SystemTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "System Time {{hostname}}",
          "refId": "G"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_OSIOWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "OS IO Wait {{hostname}}",
          "refId": "H"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_OSCPUWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "OS CPU Wait {{hostname}}",
          "refId": "I"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "OS CPU Virtual {{hostname}}",
          "refId": "J"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_ThrottlerSleepMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Throttler Sleep {{hostname}}",
          "refId": "K"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_DelayedInsertsMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000",
          "legendFormat": "Delayed Insert {{hostname}}",
          "refId": "L"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_ZooKeeperWaitMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Zookeeper Wait {{hostname}}",
          "refId": "M"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_CompileExpressionsMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Compile Expressions {{hostname}}",
          "refId": "N"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_MergesTimeMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000",
          "legendFormat": "Merges {{hostname}}",
          "refId": "O"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_RWLockReadersWaitMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000",
          "legendFormat": "RWLock Reader Wait {{hostname}}",
          "refId": "P"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_RWLockWritersWaitMilliseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000",
          "legendFormat": "RWLock Writer Wait {{hostname}}",
          "refId": "Q"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_SelectQueryTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Select Query {{hostname}}",
          "refId": "R"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_InsertQueryTimeMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "Insert Query {{hostname}}",
          "refId": "S"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_S3ReadMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "S3 Read {{hostname}}",
          "refId": "T"
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(chi_clickhouse_event_S3WriteMicroseconds{chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
          "legendFormat": "S3 Write {{hostname}}",
          "refId": "U"
        }
      ],
      "title": "CPU Time total",
      "type": "piechart"
    },
    {
      "datasource": {
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Show different types of connections for each server",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "stepAfter",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 81
      },
      "id": 48,
      "links": [
        {
          "targetBlank": true,
          "title": "max_connections",
          "url": "https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#max-connections"
        },
        {
          "targetBlank": true,
          "title": "max_distributed_connections",
          "url": "https://clickhouse.com/docs/en/operations/settings/settings#max-distributed-connections"
        },
        {
          "targetBlank": true,
          "title": "MySQL Protocol",
          "url": "https://clickhouse.com/docs/en/interfaces/mysql/"
        },
        {
          "targetBlank": true,
          "title": "HTTP Protocol",
          "url": "https://clickhouse.com/docs/en/interfaces/http/"
        },
        {
          "targetBlank": true,
          "title": "Native Protocol",
          "url": "https://clickhouse.com/docs/en/interfaces/tcp/"
        }
      ],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "10.4.3",
      "targets": [
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_TCPConnection{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "tcp {{hostname}}",
          "refId": "A",
          "step": 10
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_HTTPConnection{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "http {{hostname}}",
          "refId": "B",
          "step": 10
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_InterserverConnection{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "interserver {{hostname}}",
          "refId": "C",
          "step": 10
        },
        {
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "chi_clickhouse_metric_MySQLConnection{chi=~\"$chi\",hostname=~\"$hostname\"}",
          "intervalFactor": 2,
          "legendFormat": "mysql {{hostname}}",
          "refId": "D",
          "step": 10
        }
      ],
      "title": "Connections",
      "type": "timeseries"
    }
  ],
  "preload": false,
  "refresh": "1m",
  "schemaVersion": 40,
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {
          "text": "vm-shortterm",
          "value": "24b0d575-0cea-474c-a4eb-c225d250858d"
        },
        "includeAll": false,
        "name": "DS_PROMETHEUS",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "type": "datasource"
      },
      {
        "allValue": ".*",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, chi)",
        "includeAll": true,
        "label": "Cluster",
        "multi": true,
        "name": "chi",
        "options": [],
        "query": {
          "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, chi)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "sort": 1,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",chi=~\"$chi\"}, hostname)",
        "includeAll": true,
        "label": "Server",
        "multi": true,
        "name": "hostname",
        "options": [],
        "query": {
          "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",chi=~\"$chi\"}, hostname)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "sort": 1,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-30m",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ]
  },
  "timezone": "browser",
  "title": "Altinity ClickHouse Operator Dashboard",
  "uid": "clickhouse-operator",
  "version": 4,
  "weekStart": ""
}
