{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "iteration": 1675088603120, "links": [], "liveNow": false, "panels": [ { "datasource": { "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(245, 54, 54, 0.9)", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 50 }, { "color": "rgba(50, 172, 45, 0.97)", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, "id": 1, "links": [], "maxDataPoints": 100, "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "text": {} }, "pluginVersion": "8.5.13", "targets": [ { "expr": "(sum(up{job=\"kube-apiserver\"} == 1) / sum(up{job=\"kube-apiserver\"})) * 100", "format": "time_series", "instant": true, "intervalFactor": 1, "refId": "A", "step": 600 } ], "title": "API Servers UP", "type": "gauge" }, { "datasource": { "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(245, 54, 54, 0.9)", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 50 }, { "color": "rgba(50, 172, 45, 0.97)", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, "id": 2, "links": [], "maxDataPoints": 100, "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "text": {} }, "pluginVersion": "8.5.13", "targets": [ { "expr": "(sum(up{job=\"kube-controller-manager\"} == 1) / sum(up{job=\"kube-controller-manager\"})) * 100", "format": "time_series", "instant": true, "intervalFactor": 1, "refId": "A", "step": 600 } ], "title": "Controller Managers UP", "type": "gauge" }, { "datasource": { "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(245, 54, 54, 0.9)", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 50 }, { "color": "rgba(50, 172, 45, 0.97)", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 }, "id": 3, "links": [], "maxDataPoints": 100, "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "text": {} }, "pluginVersion": "8.5.13", "targets": [ { "expr": "(sum(up{job=\"kube-scheduler\"} == 1) / sum(up{job=\"kube-scheduler\"})) * 100", "format": "time_series", "instant": true, "intervalFactor": 1, "refId": "A", "step": 600 } ], "title": "Schedulers UP", "type": "gauge" }, { "datasource": { "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 300 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 6, "x": 18, "y": 0 }, "id": 35, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "text": {} }, "pluginVersion": "8.5.13", "targets": [ { "expr": "count(kube_namespace_created)", "hide": false, "legendFormat": "Namespaces", "refId": "N" }, { "expr": "count(kube_daemonset_created)", "hide": false, "legendFormat": "DaemonSets", "refId": "C" }, { "expr": "count(kube_deployment_created)", "hide": false, "legendFormat": "Deployments", "refId": "D" }, { "expr": "count(kube_statefulset_created)", "hide": false, "legendFormat": "StatefulSets", "refId": "M" }, { "expr": "count(kube_replicaset_created)", "hide": false, "legendFormat": "ReplicaSets", "refId": "B" }, { "expr": "count(kube_pod_created)", "hide": false, "legendFormat": "Pods", "refId": "K" }, { "expr": "count(kube_verticalpodautoscaler_labels)", "hide": false, "legendFormat": "VPA", "refId": "O" }, { "expr": "count(kube_poddisruptionbudget_created)", "hide": false, "legendFormat": "PDB", "refId": "Q" }, { "expr": "count(kube_persistentvolumeclaim_info)", "hide": false, "legendFormat": "PVC", "refId": "J" }, { "expr": "count(kube_persistentvolume_info)", "hide": false, "legendFormat": "PV", "refId": "I" }, { "expr": "count(kube_cronjob_created)", "hide": false, "legendFormat": "CronJobs", "refId": "F" }, { "expr": "count(kube_job_created)", "hide": false, "legendFormat": "Jobs", "refId": "G" }, { "expr": "count(kube_secret_created)", "hide": false, "legendFormat": "Secrets", "refId": "P" }, { "expr": "count(kube_configmap_created)", "legendFormat": "ConfigMaps", "refId": "A" }, { "expr": "count(kube_service_created)", "hide": false, "legendFormat": "Services", "refId": "L" }, { "expr": "count(kube_endpoint_created)", "hide": false, "legendFormat": "Endpoints", "refId": "E" }, { "expr": "count(kube_node_created)", "hide": false, "legendFormat": "Nodes", "refId": "H" } ], "title": "Kubernetes Resources Count", "transformations": [ { "id": "organize", "options": {} } ], "type": "bargauge" }, { "datasource": { "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 0, "y": 7 }, "id": 9, "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.5.13", "targets": [ { "expr": "count(up{job=\"kube-apiserver\"})", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "title": "API Servers count", "type": "stat" }, { "datasource": { "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 6, "y": 7 }, "id": 10, "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.5.13", "targets": [ { "expr": "count(up{job=\"kube-controller-manager\"})", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "title": "Controller Managers count", "type": "stat" }, { "datasource": { "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 7 }, "id": 11, "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.5.13", "targets": [ { "expr": "count(up{job=\"kube-scheduler\"})", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "title": "Schedulers count", "type": "stat" }, { "collapsed": false, "datasource": { "uid": "$ds_prometheus" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, "id": 46, "panels": [], "title": "Kube-apiserver", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 31, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "rate(process_cpu_seconds_total{job=\"kube-apiserver\", instance=~\"$instance:.*\"}[$__rate_interval])", "intervalFactor": 1, "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-apiserver CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:566", "format": "short", "label": "CPU Usage", "logBase": 1, "show": true }, { "$$hashKey": "object:567", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, "hiddenSeries": false, "id": 33, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "process_resident_memory_bytes{job=\"kube-apiserver\", instance=~\"$instance:.*\"}", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-apiserver Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:732", "format": "bytes", "logBase": 1, "show": true }, { "$$hashKey": "object:733", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "reqps" } ] } ] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 27, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": false, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "exemplar": true, "expr": "sum by(verb) (rate(apiserver_request_total{verb!~\"CONNECT|WATCH\", verb=~\"$verb\", instance=~\"$instance:.*\", resource=~\"$resource\"}[$__rate_interval]))", "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-apiserver requests rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:339", "format": "reqps", "logBase": 1, "show": true }, { "$$hashKey": "object:340", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.displayMode", "value": "basic" }, { "id": "displayName", "value": "RPS" }, { "id": "unit", "value": "reqps" }, { "id": "decimals", "value": 2 } ] }, { "matcher": { "id": "byName", "options": "resource" }, "properties": [ { "id": "custom.width", "value": 291 } ] }, { "matcher": { "id": "byName", "options": "verb" }, "properties": [ { "id": "custom.width", "value": 150 } ] }, { "matcher": { "id": "byName", "options": "subresource" }, "properties": [ { "id": "custom.width", "value": 100 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "id": 47, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.5.13", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "exemplar": true, "expr": "sum by(verb, resource, subresource) (rate(apiserver_request_total{verb!~\"CONNECT|WATCH\", verb=~\"$verb\", instance=~\"$instance:.*\", resource=~\"$resource\"}[$__range]))", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Kube-apiserver requests rate", "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "names": [ "resource", "verb", "Value", "subresource" ] } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Value" } ] } } ], "type": "table" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "displayName", "value": "bytes/sec" }, { "id": "unit", "value": "binBps" } ] } ] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "hiddenSeries": false, "id": 36, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": false, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "exemplar": true, "expr": "max by (verb) (rate(apiserver_response_sizes_sum{verb!~\"CONNECT|WATCH\", verb=~\"$verb\", instance=~\"$instance:.*\", resource=~\"$resource\"}[$__rate_interval])/rate(apiserver_response_sizes_count{verb!~\"CONNECT|WATCH\", verb=~\"$verb\", instance=~\"$instance:.*\", resource=~\"$resource\"}[$__rate_interval]))", "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-apiserver response sizes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:339", "format": "binBps", "logBase": 1, "show": true }, { "$$hashKey": "object:340", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.displayMode", "value": "basic" }, { "id": "displayName", "value": "bytes/s" }, { "id": "unit", "value": "binBps" }, { "id": "decimals", "value": 2 } ] }, { "matcher": { "id": "byName", "options": "resource" }, "properties": [ { "id": "custom.width", "value": 291 } ] }, { "matcher": { "id": "byName", "options": "verb" }, "properties": [ { "id": "custom.width", "value": 150 } ] }, { "matcher": { "id": "byName", "options": "subresource" }, "properties": [ { "id": "custom.width", "value": 100 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "id": 48, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.5.13", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "exemplar": true, "expr": "max by (verb, resource, subresource) (rate(apiserver_response_sizes_sum{verb!~\"CONNECT|WATCH\", verb=~\"$verb\", instance=~\"$instance:.*\", resource=~\"$resource\"}[$__rate_interval])/rate(apiserver_response_sizes_count{verb!~\"CONNECT|WATCH\", verb=~\"$verb\", instance=~\"$instance:.*\", resource=~\"$resource\"}[$__range]))", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Kube-apiserver response sizes", "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "names": [ "resource", "verb", "Value", "subresource" ] } } }, { "id": "filterByValue", "options": { "filters": [ { "config": { "id": "regex", "options": { "value": "NaN" } }, "fieldName": "Value" } ], "match": "any", "type": "exclude" } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Value" } ] } } ], "type": "table" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 29, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": false, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:1066", "alias": "/^0.[01234]/", "color": "#56A64B" }, { "$$hashKey": "object:1192", "alias": "/^0.[56789]/", "color": "#F2CC0C" }, { "$$hashKey": "object:1208", "alias": "/^[123456789]/", "color": "#E02F44" }, { "$$hashKey": "object:1216", "alias": "+Inf", "color": "#AD0317" } ], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "exemplar": true, "expr": "sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\", job=\"kube-apiserver\", instance=~\"$instance:6443\", verb=~\"$verb\"}[$__rate_interval])) by (le)", "format": "heatmap", "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-apiserver latency bucket", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:286", "format": "ops", "logBase": 1, "show": true }, { "$$hashKey": "object:287", "format": "short", "logBase": 1, "show": false } ], "yaxis": { "align": false } }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.displayMode", "value": "basic" }, { "id": "displayName", "value": "s" }, { "id": "decimals", "value": 2 }, { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "resource" }, "properties": [ { "id": "custom.width", "value": 291 } ] }, { "matcher": { "id": "byName", "options": "verb" }, "properties": [ { "id": "custom.width", "value": 150 } ] }, { "matcher": { "id": "byName", "options": "subresource" }, "properties": [ { "id": "custom.width", "value": 100 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "id": 49, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.5.13", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "exemplar": true, "expr": "max by (verb, resource, subresource) (rate(apiserver_request_duration_seconds_sum{verb!~\"CONNECT|WATCH\", subresource!~\"exec|log\", verb=~\"$verb\", instance=~\"$instance:.*\", resource=~\"$resource\"}[$__range])/rate(apiserver_request_duration_seconds_count{verb!~\"CONNECT|WATCH\", subresource!~\"exec|log\", verb=~\"$verb\", instance=~\"$instance:.*\", resource=~\"$resource\"}[$__range]))", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Kube-apiserver latency", "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "names": [ "resource", "verb", "Value", "subresource" ] } } }, { "id": "filterByValue", "options": { "filters": [ { "config": { "id": "regex", "options": { "value": "NaN" } }, "fieldName": "Value" } ], "match": "any", "type": "exclude" } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Value" } ] } } ], "type": "table" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, "hiddenSeries": false, "id": 50, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": false, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:1066", "alias": "/^0.[01234]/", "color": "#56A64B" }, { "$$hashKey": "object:1192", "alias": "/^0.[56789]/", "color": "#F2CC0C" }, { "$$hashKey": "object:1208", "alias": "/^[123456789]/", "color": "#E02F44" }, { "$$hashKey": "object:1216", "alias": "+Inf", "color": "#AD0317" } ], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "exemplar": true, "expr": "sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\", job=\"kube-apiserver\", instance=~\"$instance:6443\", verb=~\"$verb\"}[$__rate_interval])) by (le) / scalar(sum(rate(apiserver_request_duration_seconds_count{verb!~\"CONNECT|WATCH\", job=\"kube-apiserver\", instance=~\"$instance:6443\", verb=~\"$verb\"}[$__rate_interval])))", "format": "heatmap", "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-apiserver latency bucket %", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:286", "format": "percentunit", "logBase": 1, "max": "1", "min": "0", "show": true }, { "$$hashKey": "object:287", "format": "short", "logBase": 1, "show": false } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, "hiddenSeries": false, "id": 51, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\", job=\"kube-apiserver\", instance=~\"$instance:6443\", verb=~\"$verb\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-apiserver latency time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:286", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:287", "format": "short", "logBase": 1, "show": false } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, "hiddenSeries": false, "id": 52, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": false, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:1066", "alias": "/^0.00[0-9]/", "color": "#56A64B" }, { "$$hashKey": "object:1192", "alias": "/^0.0[1-9]/", "color": "#F2CC0C" }, { "$$hashKey": "object:1208", "alias": "/^0.[1-9]/", "color": "#FF780A" }, { "$$hashKey": "object:1216", "alias": "/^[1-9]/", "color": "#C4162A" }, { "$$hashKey": "object:2638", "alias": "+Inf", "color": "#AD0317" } ], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "exemplar": true, "expr": "sum(rate(etcd_request_duration_seconds_bucket{job=\"kube-apiserver\", instance=~\"$instance:6443\", verb=~\"$verb\"}[$__rate_interval])) by (le)", "format": "heatmap", "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Etcd requests latency bucket", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:286", "format": "ops", "logBase": 1, "show": true }, { "$$hashKey": "object:287", "format": "short", "logBase": 1, "show": false } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, "hiddenSeries": false, "id": 53, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket{job=\"kube-apiserver\", instance=~\"$instance:6443\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Etcd latency time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:286", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:287", "format": "short", "logBase": 1, "show": false } ], "yaxis": { "align": false } }, { "collapsed": false, "datasource": { "uid": "$ds_prometheus" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 }, "id": 44, "panels": [], "title": "Kube-scheduler", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 60 }, "hiddenSeries": false, "id": 37, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "rate(process_cpu_seconds_total{job=\"kube-scheduler\", instance=~\"$instance:.*\"}[$__rate_interval])", "intervalFactor": 1, "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-scheduler CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:566", "format": "short", "label": "CPU Usage", "logBase": 1, "show": true }, { "$$hashKey": "object:567", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 60 }, "hiddenSeries": false, "id": 38, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "process_resident_memory_bytes{job=\"kube-scheduler\", instance=~\"$instance:.*\"}", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-scheduler Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:732", "format": "bytes", "logBase": 1, "show": true }, { "$$hashKey": "object:733", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": false, "datasource": { "uid": "$ds_prometheus" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 }, "id": 42, "panels": [], "title": "Kube-controller-manager", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 69 }, "hiddenSeries": false, "id": 39, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "rate(process_cpu_seconds_total{job=\"kube-controller-manager\", instance=~\"$instance:.*\"}[$__rate_interval])", "intervalFactor": 1, "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-controller-manager CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:566", "format": "short", "label": "CPU Usage", "logBase": 1, "show": true }, { "$$hashKey": "object:567", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$ds_prometheus" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 69 }, "hiddenSeries": false, "id": 40, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.5.13", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "process_resident_memory_bytes{job=\"kube-controller-manager\", instance=~\"$instance:.*\"}", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kube-controller-manager Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:732", "format": "bytes", "logBase": 1, "show": true }, { "$$hashKey": "object:733", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "refresh": "30s", "schemaVersion": 36, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "main", "value": "main" }, "hide": 0, "includeAll": false, "label": "Prometheus", "multi": false, "name": "ds_prometheus", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "definition": "label_values(up{job=\"kube-apiserver\"}, instance)", "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", "options": [], "query": { "query": "label_values(up{job=\"kube-apiserver\"}, instance)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "/(.*):.*/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "definition": "label_values(apiserver_request_total, resource)", "hide": 0, "includeAll": true, "label": "Resource", "multi": true, "name": "resource", "options": [], "query": { "query": "label_values(apiserver_request_total, resource)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "definition": "label_values(apiserver_request_total{verb!~\"CONNECT|WATCH\"}, verb)", "hide": 0, "includeAll": true, "label": "VERB", "multi": true, "name": "verb", "options": [], "query": { "query": "label_values(apiserver_request_total{verb!~\"CONNECT|WATCH\"}, verb)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "Control Plane Status", "uid": "3yqtXDzia", "version": 1, "weekStart": "" }