{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": false, "fiscalYearStartMonth": 0, "gnetId": null, "graphTooltip": 1, "iteration": 1638543860824, "links": [], "liveNow": false, "panels": [ { "columns": [], "datasource": "$ds_prometheus", "description": "Note that this table shows the average values for the entire period selected in the dashboard. Consequently, it may contain information about Pods or namespaces that were changed or deleted during the selected period.", "fieldConfig": { "defaults": { "custom": { "align": null, "displayMode": "auto", "filterable": false, "minWidth": 50 }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "namespace" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null }, { "id": "displayName", "value": "Namespace" }, { "id": "custom.width" }, { "id": "links", "value": [ { "targetBlank": true, "title": "Namespace Details", "url": "/d/sZzUB4ymk1/namespace?var-ds_prometheus=${ds_prometheus}&var-namespace=${__value.raw}&${__url_time_range}" } ] }, { "id": "custom.minWidth", "value": 280 } ] }, { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [ { "id": "displayName", "value": "VPA %" }, { "id": "unit", "value": "percent" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null }, { "id": "mappings", "value": [] } ] }, { "matcher": { "id": "byName", "options": "Value #C" }, "properties": [ { "id": "displayName", "value": "CPU" }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 3 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #D" }, "properties": [ { "id": "displayName", "value": "Req CPU" }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #E" }, "properties": [ { "id": "displayName", "value": "Over-req CPU" }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #F" }, "properties": [ { "id": "displayName", "value": "Under-req CPU" }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #G" }, "properties": [ { "id": "displayName", "value": "Throttling cores" }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #H" }, "properties": [ { "id": "displayName", "value": "Memory" }, { "id": "unit", "value": "bytes" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #I" }, "properties": [ { "id": "displayName", "value": "Req Memory" }, { "id": "unit", "value": "bytes" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #J" }, "properties": [ { "id": "displayName", "value": "Over-req Memory" }, { "id": "unit", "value": "bytes" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #K" }, "properties": [ { "id": "displayName", "value": "Under-req Memory" }, { "id": "unit", "value": "bytes" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #L" }, "properties": [ { "id": "displayName", "value": "RX Network" }, { "id": "unit", "value": "Bps" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #M" }, "properties": [ { "id": "displayName", "value": "TX Network" }, { "id": "unit", "value": "Bps" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #N" }, "properties": [ { "id": "displayName", "value": "Read IOPS" }, { "id": "unit", "value": "iops" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #O" }, "properties": [ { "id": "displayName", "value": "Write IOPS" }, { "id": "unit", "value": "iops" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #P" }, "properties": [ { "id": "displayName", "value": "Pod Restarts" }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #Q" }, "properties": [ { "id": "displayName", "value": "Write IOPS" }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #R" }, "properties": [ { "id": "displayName", "value": "Total replicas count" }, { "id": "unit", "value": "short" }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #S" }, "properties": [ { "id": "displayName", "value": "Not ready replicas" }, { "id": "unit", "value": "short" }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Value #T" }, "properties": [ { "id": "displayName", "value": "Pod Restarts" }, { "id": "unit", "value": "short" }, { "id": "custom.align", "value": null } ] }, { "matcher": { "id": "byName", "options": "Controller" }, "properties": [ { "id": "custom.width", "value": 212 } ] } ] }, "fontSize": "100%", "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "id": 4, "links": [], "options": { "showHeader": true }, "pageSize": 50, "pluginVersion": "8.2.3", "scroll": true, "showHeader": true, "sort": { "col": 4, "desc": true }, "styles": [ { "alias": "Time", "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "hidden" }, { "alias": "Namespace", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "link": true, "linkTargetBlank": true, "linkTooltip": "Namespace", "linkUrl": "/d/sZzUB4ymk1/namespace?var-ds_prometheus=${ds_prometheus}&var-namespace=${__value.raw}&${__url_time_range}", "mappingType": 1, "pattern": "namespace", "thresholds": [], "type": "string", "unit": "short" }, { "alias": "VPA %", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #B", "thresholds": [], "type": "number", "unit": "percent" }, { "alias": "CPU", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #C", "thresholds": [], "type": "number", "unit": "short" }, { "alias": "Req CPU", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 3, "mappingType": 1, "pattern": "Value #D", "thresholds": [], "type": "number", "unit": "short" }, { "alias": "Over-req CPU", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #E", "thresholds": [], "type": "number", "unit": "short" }, { "alias": "Under-req CPU", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #F", "thresholds": [], "type": "number", "unit": "short" }, { "alias": "Throttling cores", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #G", "thresholds": [], "type": "number", "unit": "short" }, { "alias": "Memory", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #H", "thresholds": [], "type": "number", "unit": "bytes" }, { "alias": "Req Memory", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #I", "thresholds": [], "type": "number", "unit": "bytes" }, { "alias": "Over-req Memory", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #J", "thresholds": [], "type": "number", "unit": "bytes" }, { "alias": "Under-req Memory", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #K", "thresholds": [], "type": "number", "unit": "bytes" }, { "alias": "RX Network", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #L", "thresholds": [], "type": "number", "unit": "Bps" }, { "alias": "TX Network", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #M", "thresholds": [], "type": "number", "unit": "Bps" }, { "alias": "Read IOPS", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #N", "thresholds": [], "type": "number", "unit": "iops" }, { "alias": "Write IOPS", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #O", "thresholds": [], "type": "number", "unit": "iops" }, { "alias": "Pod Restarts", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 0, "mappingType": 1, "pattern": "Value #P", "thresholds": [], "type": "number", "unit": "short" }, { "alias": "", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "/.*/", "thresholds": [], "type": "hidden", "unit": "short" } ], "targets": [ { "expr": "max by (namespace) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range]))", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "Namespace", "refId": "A" }, { "expr": "100 * count by (namespace) (\n sum by (namespace, verticalpodautoscaler) ( \n count by (namespace, controller_name, verticalpodautoscaler) (avg_over_time(vpa_target_recommendation{namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\n / on (controller_name, namespace) group_left\n count by (namespace, controller_name) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range]))\n )\n) \n/ count by (namespace) (sum by (namespace, controller) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "VPA %", "refId": "B" }, { "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "CPU", "refId": "C" }, { "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\nor\ncount(avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Req CPU", "refId": "D" }, { "expr": "sum by (namespace)\n (\n (\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n -\n sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n ) > 0\n )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Over-req CPU", "refId": "E" }, { "expr": "sum by (namespace)\n (\n (\n (\n sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n -\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n )\n > 0\n )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Under-req CPU", "refId": "F" }, { "expr": "(sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) / sum by (namespace) (rate(container_cpu_cfs_periods_total{node=~\"$node\", namespace=~\"$namespace\"}[$__range]))) * sum by (namespace) (kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"})\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Throttling cores", "refId": "G" }, { "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Memory", "refId": "H" }, { "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\nor\ncount(avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Req Memory", "refId": "I" }, { "expr": "sum by (namespace)\n (\n (\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n -\n sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n ) > 0\n )\nor\ncount(avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Over-req Memory", "refId": "J" }, { "expr": "sum by (namespace)\n (\n (\n (\n sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n -\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n )\n > 0\n )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Under-req Memory", "refId": "K" }, { "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) \n- \n(\n sum by (namespace) (\n sum by (namespace, pod) (rate(container_network_receive_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n * \n sum by (namespace, pod) (avg_over_time(kube_pod_info{host_network=\"true\",namespace=~\"$namespace\"}[$__range]))\n )\n)\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "RX Network", "refId": "L" }, { "expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) \n- \n(\n sum by (namespace) (\n sum by (namespace, pod) (rate(container_network_transmit_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n * \n sum by (namespace, pod) (avg_over_time(kube_pod_info{host_network=\"true\",namespace=~\"$namespace\"}[$__range]))\n )\n)\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "TX Network", "refId": "M" }, { "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "Read IOPS", "refId": "N" }, { "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "Write IOPS", "refId": "O" }, { "expr": "sum by (namespace) (increase(kube_pod_container_status_restarts_total{node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Restart count", "refId": "P" } ], "title": "Average", "transform": "table", "transformations": [ { "id": "merge", "options": {} }, { "id": "filterFieldsByName", "options": { "include": { "names": [ "namespace", "Value #B", "Value #C", "Value #D", "Value #E", "Value #F", "Value #G", "Value #H", "Value #I", "Value #J", "Value #K", "Value #L", "Value #M", "Value #N", "Value #O", "Value #P" ] } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "namespace" } ] } } ], "type": "table" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "decimals": 0, "description": "The number of Pods in each namespace", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, "y": 8 }, "hiddenSeries": false, "id": 332, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": false, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "count by (namespace) (max_over_time(kube_pod_info{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "B" }, { "expr": "count(kube_pod_info{node=~\"$node\", namespace=~\"$namespace\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pods count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "decimals": 0, "description": "The number of Pod restarts", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 12, "y": 8 }, "hiddenSeries": false, "id": 564, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": false, "sort": "avg", "sortDesc": true, "total": false, "values": false }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) (increase(kube_pod_container_status_restarts_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "B" }, { "expr": "sum (increase(kube_pod_container_status_restarts_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pods restarts", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, "id": 8, "panels": [], "title": "CPU", "type": "row" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 6, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "A" }, { "expr": "sum (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Usage by namespace", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "cores", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "The total value may not equal the sum of system and user times because of the kernel's cgroup accounting peculiarities. Read more here: https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 13 }, "hiddenSeries": false, "id": 10, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false }, { "alias": "System", "color": "#e24d42" }, { "alias": "User", "color": "#1f78c1" } ], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "expr": "sum (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "System", "refId": "A" }, { "expr": "sum (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "User", "refId": "B" }, { "expr": "sum (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Usage by state", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "cores", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "This graph shows the requested CPU resources higher than the actual CPU consumption. In other words, it shows CPU resources that can be \"freed\" without affecting the service.", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 22 }, "hiddenSeries": false, "id": 333, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "expr": "sum by (namespace)\n (\n (\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n -\n sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n ) > 0\n )", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "B" }, { "expr": "sum\n(\n (\n (\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n -\n sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n ) > 0\n )\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Over-requested by namespace", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "cores", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "This graph shows the requested CPU resources lower than the actual CPU consumption. In other words, it shows CPU resources that need to be \"reserved\" for the service to run smoothly.", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 22 }, "hiddenSeries": false, "id": 386, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "expr": "sum by (namespace)\n (\n (\n (\n sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n -\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n )\n > 0\n )", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "B" }, { "expr": "sum\n (\n (\n (\n sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n -\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n )\n > 0\n )", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Under-requested by namespace", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "cores", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "The absence of data on the graph means that container resources are not set", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 315, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) ((sum by (namespace, pod) (rate(container_cpu_cfs_throttled_periods_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by (namespace, pod) (rate(container_cpu_cfs_periods_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))) * sum by (namespace, pod) (kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "A" }, { "expr": "sum(sum by (namespace) ((sum by (namespace, pod) (rate(container_cpu_cfs_throttled_periods_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by (namespace, pod) (rate(container_cpu_cfs_periods_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))) * sum by (namespace, pod) (kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"})))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Throttling", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "cores", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, "id": 343, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "", "fill": 0, "gridPos": { "h": 9, "w": 6, "x": 0, "y": 40 }, "id": 352, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "namespace", "repeatDirection": "h", "seriesOverrides": [ { "alias": "Usage", "color": "#629e51" }, { "alias": "Requests", "color": "#f4d598" }, { "alias": "Limits", "color": "#c15c17" }, { "alias": "VPA Target", "color": "#447ebc" } ], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Usage", "refId": "A" }, { "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "Requests", "refId": "B" }, { "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limits", "refId": "C" }, { "expr": "sum by (namespace) (avg_over_time(vpa_target_recommendation{container!=\"POD\", namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "VPA Target", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "$namespace", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "cores", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Namespaces CPU", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, "id": 12, "panels": [ { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "The total value may not equal the sum of system and user times because of the kernel's cgroup accounting peculiarities. Read more here: https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt", "fill": 1, "gridPos": { "h": 9, "w": 6, "x": 0, "y": 41 }, "id": 14, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": false }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "namespace", "repeatDirection": "h", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false }, { "alias": "System", "color": "#e24d42" }, { "alias": "User", "color": "#1f78c1" } ], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "expr": "sum by (namespace) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "System", "refId": "A" }, { "expr": "sum by (namespace) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "User", "refId": "B" }, { "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "$namespace", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "cores", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "Namespaces CPU by state", "type": "row" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, "id": 38, "panels": [], "title": "Memory", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 37 }, "hiddenSeries": false, "id": 40, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": false, "sort": "avg", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "lines": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "A" }, { "expr": "sum (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Usage by namespace", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "The Working set bytes metric is the actual memory used by the container, as it includes active file memory. When its value approaches the limit, the container can be killed by the OOMKiller. This value can be higher than the sum RSS and Cache since not all active file memory is Cache.", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 37 }, "hiddenSeries": false, "id": 41, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": false, "sort": "avg", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Working set bytes without kmem", "color": "rgb(0, 0, 0)", "dashes": true, "fill": 0, "linewidth": 2, "stack": false }, { "alias": "Kmem", "color": "rgb(255, 0, 0)", "dashes": true, "fill": 0, "linewidth": 2, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum (avg_over_time(container_memory_rss{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "RSS", "refId": "A" }, { "expr": "sum (avg_over_time(container_memory_cache{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Cache", "refId": "B" }, { "expr": "sum (avg_over_time(container_memory_swap{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Swap", "refId": "C" }, { "expr": "sum (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Working set bytes without kmem", "refId": "D" }, { "expr": "sum (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Kmem", "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Usage by state", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "This graph shows the requested Memory resources higher than the actual Memory consumption. In other words, it shows Memory resources that can be \"freed\" without affecting the service.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 46 }, "hiddenSeries": false, "id": 361, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": false, "sort": "avg", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "lines": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace)\n (\n (\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n -\n sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n ) > 0\n )", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "A" }, { "expr": "sum\n (\n (\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n -\n sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n ) > 0\n )", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Over-requested by namespace", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "This graph shows the requested Memory resources higher than the actual Memory consumption. In other words, it shows Memory resources that need to be \"reserved\" for the service to run smoothly.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 46 }, "hiddenSeries": false, "id": 387, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": false, "sort": "avg", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "lines": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace)\n (\n (\n (\n sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n -\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n )\n > 0\n )", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "A" }, { "expr": "sum\n (\n (\n (\n sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n -\n sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n )\n > 0\n )", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Under-requested by namespace", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 53 }, "id": 43, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "The Working set bytes metric is the actual memory used by the container, as it includes active file memory. When its value approaches the limit, the container can be killed by the OOMKiller. This value can be higher than the sum RSS and Cache since not all active file memory is Cache.", "fill": 1, "gridPos": { "h": 9, "w": 6, "x": 0, "y": 59 }, "id": 44, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "namespace", "repeatDirection": "h", "seriesOverrides": [ { "alias": "Working set bytes without kmem", "color": "rgb(0, 0, 0)", "dashes": true, "fill": 0, "linewidth": 2, "stack": false }, { "alias": "Kmem", "color": "rgb(255, 0, 0)", "dashes": true, "fill": 0, "linewidth": 2, "stack": false }, { "alias": "VPA Target", "color": "#447ebc", "dashes": true, "fill": 0, "lines": false, "linewidth": 2, "stack": false }, { "alias": "Requests", "color": "#f4d598", "dashes": true, "fill": 0, "lines": false, "linewidth": 2, "stack": false }, { "alias": "Limits", "color": "#c15c17", "dashes": true, "fill": 0, "lines": false, "linewidth": 2, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "RSS", "refId": "A" }, { "expr": "sum by (namespace) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Cache", "refId": "B" }, { "expr": "sum by (namespace) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Swap", "refId": "C" }, { "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Working set bytes without kmem", "refId": "D" }, { "expr": "sum by(namespace) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"memory\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "VPA Target", "refId": "E" }, { "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Requests", "refId": "F" }, { "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limits", "refId": "G" }, { "expr": "sum by (namespace) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Kmem", "refId": "H" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "$namespace", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Namespaces Memory", "type": "row" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 54 }, "id": 77, "panels": [], "title": "Network", "type": "row" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "This graph shows Network Receive (except for the hostNetwork Pods)", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, "hiddenSeries": false, "id": 79, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])) \n- \n(\n sum by (namespace) (\n sum by (namespace, pod) (rate(container_network_receive_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n * \n sum by (namespace, pod) (kube_pod_info{host_network=\"true\",namespace=~\"$namespace\"})\n )\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "A" }, { "expr": "sum (\n sum by (namespace) (rate(container_network_receive_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])) \n - \n (\n sum by (namespace) (\n sum by (namespace, pod) (rate(container_network_receive_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n * \n sum by (namespace, pod) (kube_pod_info{host_network=\"true\",namespace=~\"$namespace\"})\n )\n )\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Receive", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "This graph shows Network Transmit (except for the hostNetwork Pods)", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, "hiddenSeries": false, "id": 276, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])) \n- \n(\n sum by (namespace) (\n sum by (namespace, pod) (rate(container_network_transmit_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n * \n sum by (namespace, pod) (kube_pod_info{host_network=\"true\",namespace=~\"$namespace\"})\n )\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "A" }, { "expr": "sum (\n sum by (namespace) (rate(container_network_transmit_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])) \n - \n (\n sum by (namespace) (\n sum by (namespace, pod) (rate(container_network_transmit_bytes_total{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n * \n sum by (namespace, pod) (kube_pod_info{host_network=\"true\",namespace=~\"$namespace\"})\n )\n )\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Transmit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 63 }, "id": 154, "panels": [], "title": "IOPS", "type": "row" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 64 }, "hiddenSeries": false, "id": 156, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "A" }, { "expr": "sum (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "iops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 64 }, "hiddenSeries": false, "id": 277, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Total", "bars": false, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ namespace }}", "refId": "A" }, { "expr": "sum (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "iops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 72 }, "id": 518, "panels": [], "title": "PVC", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "This graph does not show any localstorage-related information due to incorrectly calculating the occupied space on localstorage disks", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 73 }, "hiddenSeries": false, "id": 519, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Total/", "fill": 0, "linewidth": 2, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "(sum by (namespace) (\n (\n sum by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{node=~\"$node\"})\n * on (namespace, persistentvolumeclaim)\n sum by (namespace, persistentvolume, persistentvolumeclaim) (\n label_replace(\n max_over_time(kube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]),\n \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n )\n )\n or on(namespace, persistentvolumeclaim) (\n sum by (persistentvolume) (\n max_over_time(kube_persistentvolume_capacity_bytes[$__rate_interval])\n )\n * on(persistentvolume) group_left(namespace, persistentvolumeclaim)\n sum by (namespace, persistentvolume, persistentvolumeclaim) (\n label_replace(\n max_over_time(kube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]),\n \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n )\n )\n) and on (namespace) (kube_namespace_created * scalar(count (kube_node_info) == bool count (kube_node_info{node=~\"$node\"})) > 0))\nor sum by (namespace) (\n sum by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{node=~\"$node\"})\n * on (namespace, persistentvolumeclaim)\n sum by (namespace, persistentvolume, persistentvolumeclaim) (\n label_replace(\n max_over_time(kube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]),\n \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n )\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Provisioned {{ namespace }}", "refId": "A" }, { "expr": "sum(\n(sum by (namespace) (\n (\n sum by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{node=~\"$node\"})\n * on (namespace, persistentvolumeclaim)\n sum by (namespace, persistentvolume, persistentvolumeclaim) (\n label_replace(\n max_over_time(kube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]),\n \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n )\n )\n or on(namespace, persistentvolumeclaim) (\n sum by (persistentvolume) (\n max_over_time(kube_persistentvolume_capacity_bytes[$__rate_interval])\n )\n * on(persistentvolume) group_left(namespace, persistentvolumeclaim)\n sum by (namespace, persistentvolume, persistentvolumeclaim) (\n label_replace(\n max_over_time(kube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]),\n \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n )\n )\n) and on (namespace) (kube_namespace_created * scalar(count (kube_node_info) == bool count (kube_node_info{node=~\"$node\"})) > 0))\nor sum by (namespace) (\n sum by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{node=~\"$node\"})\n * on (namespace, persistentvolumeclaim)\n sum by (namespace, persistentvolume, persistentvolumeclaim) (\n label_replace(\n max_over_time(kube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]),\n \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n )\n))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PVC Provisioned (except local storage classes)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "This graph does not show any localstorage-related information due to incorrectly calculating the occupied space on localstorage disks", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 73 }, "hiddenSeries": false, "id": 520, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Total/", "fill": 0, "linewidth": 2, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) (\n sum by (namespace, persistentvolumeclaim) (\n max_over_time(kubelet_volume_stats_capacity_bytes{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]) - max_over_time(kubelet_volume_stats_available_bytes{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])\n )\n * on (namespace, persistentvolumeclaim) group_left(persistentvolume)\n\tsum by (namespace, persistentvolume, persistentvolumeclaim) (\n\t\tlabel_replace(\n\t\t\tmax_over_time(kube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]),\n\t\t\t\"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n\t\t\t+ on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n\t)\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Usage {{ namespace }}", "refId": "A" }, { "expr": "sum(\n sum by (namespace) (\n sum by (namespace, persistentvolumeclaim) (\n max_over_time(kubelet_volume_stats_capacity_bytes{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]) - max_over_time(kubelet_volume_stats_available_bytes{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])\n )\n \t* on (namespace, persistentvolumeclaim) group_left(persistentvolume)\n\t\tsum by (namespace, persistentvolume, persistentvolumeclaim) (\n\t\t\tlabel_replace(\n\t\t\t\tmax_over_time(kube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]), \n\t\t\t\t\"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n\t\t\t\t+ on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n\t\t)\n )\n)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Total", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PVC Used (except local storage classes)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$ds_prometheus", "description": "This graph does not show any localstorage-related information due to incorrectly calculating the occupied space on localstorage disks", "fill": 1, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 80 }, "id": 521, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Total/", "fill": 0, "linewidth": 2, "stack": false } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (namespace) (\n sum by (namespace, persistentvolumeclaim) (\n max_over_time(kubelet_volume_stats_capacity_bytes{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]) - max_over_time(kubelet_volume_stats_available_bytes{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])\n )\n * on (namespace, persistentvolumeclaim)\n\tsum by (namespace, persistentvolumeclaim) (\n\t\tlabel_replace(\n\t\t\tmax_over_time(\n\t\t\t\tkube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]),\n\t\t\t\t\"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n\t\t\t\t+ on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n\t)\n)\n/\nsum by (namespace) (\n sum by (namespace, persistentvolumeclaim) (\n max_over_time(kubelet_volume_stats_capacity_bytes{node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval])\n )\n * on (namespace, persistentvolumeclaim)\n\tsum by (namespace, persistentvolumeclaim) (\n\t\tlabel_replace(\n\t\t\tmax_over_time(\n\t\t\t\tkube_persistentvolumeclaim_info{namespace=~\"$namespace\"}[$__rate_interval]),\n\t\t\t\t\"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n\t\t\t\t+ on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0)\n\t)\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Usage {{ namespace }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PVC Used in % (except local storage classes)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "30s", "schemaVersion": 32, "style": "dark", "tags": [ "main" ], "templating": { "list": [ { "current": { "selected": false, "text": "default", "value": "default" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": "Prometheus", "multi": false, "name": "ds_prometheus", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$ds_prometheus", "definition": "label_values(kubernetes_build_info, node)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": "Node", "multi": true, "name": "node", "options": [], "query": { "query": "label_values(kubernetes_build_info, node)", "refId": "main-node-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": "", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$ds_prometheus", "definition": "label_values(kube_pod_info{node=~\"$node\"}, namespace)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": "Namespace", "multi": true, "name": "namespace", "options": [], "query": { "query": "label_values(kube_pod_info{node=~\"$node\"}, namespace)", "refId": "main-namespace-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "Namespaces", "uid": "V_WjJ-Pmk", "version": 2 }