{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "", "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 20, "iteration": 1663071955430, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 0, "y": 0 }, "id": 19, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "expr": "sum(kube_node_status_condition{node=~\"$node\", condition=\"Ready\", status=\"true\"} == 1)", "refId": "A" } ], "title": "Ready Nodes", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 3, "y": 0 }, "id": 20, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "expr": "sum(kube_node_info{node=~\"$node\"})", "refId": "A" } ], "title": "All Nodes", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 6, "y": 0 }, "id": 27, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "expr": "sum(max by (namespace) (kube_namespace_status_phase{namespace=~\"$namespace\", phase=\"Active\"}))", "refId": "A" } ], "title": "Namespaces", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 6, "w": 9, "x": 11, "y": 0 }, "id": 25, "options": { "content": "✓ If you have pods in pending state in your cluster, it can be caused because the scheduler cannot find a node with enough memory and CPU allocatable.
\n✓ Check that you have at least 1 node in your cluster with enough allocatable CPU and memory to accomplish the requests of the pending pods. Be sure that the node with enough resources has not a taint or is not unscheduleable.
\n✓ Check that that node has pods available to schedule in it.
\n✓ If the nodes ran out of allocatable resources you can try to adjust the requests of your pods to optimize the capacity of your nodes using the Memory Allocation Optimization and CPU Allocation Optimization dashboards. If this is not possible, you will have to increase the number of nodes in your cluster.
\n✓ If there are enough resources in the nodes, go to the Main folder dashboards and check the available resource in the quotas of the namespace.", "mode": "markdown" }, "pluginVersion": "8.5.9", "title": "Tips", "transparent": true, "type": "text" }, { "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 44, "title": "Capacity", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 3, "x": 0, "y": 7 }, "id": 42, "links": [], "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})) / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "% Requested CPU", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 3, "x": 3, "y": 7 }, "id": 41, "links": [], "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})) / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",unit=\"byte\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "% Requested Memory", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "How much CPU is actually used in the cluster (CPU usage / Allocatable CPU).", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 3, "x": 7, "y": 7 }, "id": 46, "links": [], "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval])))\n / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "% Actual CPU Usage", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "How much memory is actually used in the cluster (Memory usage / Allocatable Memory).", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 3, "x": 10, "y": 7 }, "id": 45, "links": [], "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"})) / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",unit=\"byte\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "% Actual Memory Usage", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "How many requested cores are used (Actual CPU / Requested CPU).", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 3, "x": 14, "y": 7 }, "id": 47, "links": [], "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval]))) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "% Used Requested CPU", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "How much requested memory is used (Actual Memory / Requested Memory). ", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 3, "x": 17, "y": 7 }, "id": 48, "links": [], "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"} )) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" } ], "title": "% Used Requested Memory", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "Pods number in the cluster / Allocatable Pods.", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 3, "x": 21, "y": 7 }, "id": 40, "links": [], "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(((max by (pod, namespace)(kube_pod_container_status_running))==1) * on (pod, namespace) group_right() (kube_pod_info{node=~\"$node\"}==1) \n) / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"pods\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "% Pods Usage", "transparent": true, "type": "gauge" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 34, "panels": [], "title": "Allocatable / Requested / Unused", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "This represents the total [CPU resource requests](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu) in the cluster.\nFor comparison the total [allocatable CPU cores](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node-allocatable.md) is also shown.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.stacking", "value": { "group": "A", "mode": "none" } } ] } ] }, "gridPos": { "h": 11, "w": 8, "x": 0, "y": 16 }, "id": 1, "links": [], "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{ node }}", "range": true, "refId": "A", "step": 20 }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Allocatable CPU", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.stacking", "value": { "group": "A", "mode": "none" } } ] } ] }, "gridPos": { "h": 11, "w": 8, "x": 8, "y": 16 }, "id": 28, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",node=~\"$node\"}[$__rate_interval])) - on(node) (sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"}))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{ node }}", "range": true, "refId": "A", "step": 20 }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",node=~\"$node\"}[$__rate_interval])) - on(node) (sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Unrequested CPU", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "Difference between requested cores and used cores.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.stacking", "value": { "group": "A", "mode": "none" } } ] } ] }, "gridPos": { "h": 11, "w": 8, "x": 16, "y": 16 }, "id": 31, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0\n", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{ node }}", "range": true, "refId": "A", "step": 20 }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0)", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Unused Requested CPU (Idle)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "This represents the total [memory resource requests](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory) in the cluster.\nFor comparison the total [allocatable memory](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node-allocatable.md) is also shown.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.stacking", "value": { "group": "A", "mode": "none" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 11, "w": 8, "x": 0, "y": 27 }, "id": 23, "links": [], "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",unit=\"byte\",node=~\"$node\"}[$__rate_interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{ node }}", "range": true, "refId": "A", "step": 20 }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",unit=\"byte\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Allocatable Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.stacking", "value": { "group": "A", "mode": "none" } } ] } ] }, "gridPos": { "h": 11, "w": 8, "x": 8, "y": 27 }, "id": 29, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",node=~\"$node\"}[$__rate_interval])) - on(node) (sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"}))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{ node }}", "range": true, "refId": "A", "step": 20 }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",node=~\"$node\"}[$__rate_interval])) - on(node) (sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Unrequested Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "Difference between requested memory and used memory.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.stacking", "value": { "group": "A", "mode": "none" } } ] } ] }, "gridPos": { "h": 11, "w": 8, "x": 16, "y": 27 }, "id": 32, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0\n", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{ node }}", "range": true, "refId": "A", "step": 20 }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum((sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0)", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Unused Requested Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.stacking", "value": { "group": "A", "mode": "none" } } ] } ] }, "gridPos": { "h": 11, "w": 13, "x": 0, "y": 38 }, "id": 26, "links": [], "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"pods\",node=~\"$node\"}[$__rate_interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{ node }}", "range": true, "refId": "A", "step": 20 }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"pods\",node=~\"$node\"}[$__rate_interval])))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Allocatable Pods", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.stacking", "value": { "group": "A", "mode": "none" } } ] } ] }, "gridPos": { "h": 11, "w": 11, "x": 13, "y": 38 }, "id": 30, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"pods\",node=~\"$node\"}[$__rate_interval])) - on(node) (count by (node) (\n ((max by (pod, namespace)(kube_pod_container_status_running))==1) * on (pod, namespace) group_right() (kube_pod_info{node=~\"$node\"}==1) \n))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{ node }}", "range": true, "refId": "A", "step": 20 }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "expr": "sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"pods\",node=~\"$node\"}[$__rate_interval])) - on(node) (count by (node) (\n ((max by (pod, namespace)(kube_pod_container_status_running))==1) * on (pod, namespace) group_right() (kube_pod_info{node=~\"$node\"}==1) \n)))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Available Pod slots", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 49 }, "id": 39, "panels": [], "title": "Top", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 0.5 }, { "color": "dark-orange", "value": 1 }, { "color": "red", "value": 1.5 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.displayMode", "value": "color-background" } ] } ] }, "gridPos": { "h": 12, "w": 12, "x": 0, "y": 50 }, "id": 36, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "exemplar": false, "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) * -1 > 0)\n", "format": "table", "instant": true, "range": false, "refId": "A" } ], "title": "Top 10 CPU overrequested containers", "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "names": [ "container", "namespace", "pod", "Value" ] } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Value": 3, "container": 2, "namespace": 0, "pod": 1 }, "renameByName": { "Value": "Cores", "container": "Container", "namespace": "Namespace", "pod": "Pod" } } } ], "transparent": true, "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 500000000 }, { "color": "#EAB839", "value": 1000000000 }, { "color": "red", "value": 2000000000 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.displayMode", "value": "color-background" }, { "id": "unit", "value": "bytes" } ] } ] }, "gridPos": { "h": 12, "w": 12, "x": 12, "y": 50 }, "id": 37, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "exemplar": false, "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) * -1 >0)\n", "format": "table", "instant": true, "range": false, "refId": "A" } ], "title": "Top 10 Memory overrequested containers", "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "names": [ "container", "namespace", "pod", "Value" ] } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Value": 3, "container": 2, "namespace": 0, "pod": 1 }, "renameByName": { "Value": "Bytes", "container": "Container", "namespace": "Namespace", "pod": "Pod" } } } ], "transparent": true, "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 0.5 }, { "color": "dark-orange", "value": 1 }, { "color": "red", "value": 1.5 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.displayMode", "value": "color-background" } ] } ] }, "gridPos": { "h": 12, "w": 12, "x": 0, "y": 62 }, "id": 49, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "exemplar": false, "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) > 0)\n", "format": "table", "instant": true, "range": false, "refId": "A" } ], "title": "Top 10 CPU underrequested containers", "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "names": [ "container", "namespace", "pod", "Value" ] } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Value": 3, "container": 2, "namespace": 0, "pod": 1 }, "renameByName": { "Value": "Cores", "container": "Container", "namespace": "Namespace", "pod": "Pod" } } } ], "transparent": true, "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 500000000 }, { "color": "#EAB839", "value": 1000000000 }, { "color": "red", "value": 2000000000 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.displayMode", "value": "color-background" }, { "id": "unit", "value": "bytes" } ] } ] }, "gridPos": { "h": 12, "w": 12, "x": 12, "y": 62 }, "id": 50, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.5.9", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "editorMode": "code", "exemplar": false, "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) >0)\n", "format": "table", "instant": true, "range": false, "refId": "A" } ], "title": "Top 10 Memory underrequested containers", "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "names": [ "container", "namespace", "pod", "Value" ] } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Value": 3, "container": 2, "namespace": 0, "pod": 1 }, "renameByName": { "Value": "Bytes", "container": "Container", "namespace": "Namespace", "pod": "Pod" } } } ], "transparent": true, "type": "table" } ], "refresh": "30s", "schemaVersion": 36, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "default", "value": "default" }, "hide": 0, "includeAll": false, "label": "Prometheus", "multi": false, "name": "ds_prometheus", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": "", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "definition": "label_values(up{job=\"node-exporter\"}, node)", "hide": 0, "includeAll": true, "label": "Nodes", "multi": true, "name": "node", "options": [], "query": { "query": "label_values(up{job=\"node-exporter\"}, node)", "refId": "main-node-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": "", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "definition": "label_values(kube_pod_info{node=~\"$node\"}, namespace)", "hide": 0, "includeAll": true, "label": "Namespace", "multi": true, "name": "namespace", "options": [], "query": { "query": "label_values(kube_pod_info{node=~\"$node\"}, namespace)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "filters": [], "hide": 0, "name": "Filters", "skipUrlSync": false, "type": "adhoc" } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Capacity Planning", "uid": "Tf3tuvziz1fds", "version": 1, "weekStart": "" }