{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "datasource",
          "uid": "grafana"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "editable": false,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": 117,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "Note that this table shows the average values for the entire period selected in the dashboard. Consequently, it may contain information about Pods or namespaces that were changed or deleted during the selected period.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "type": "auto"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 75
          },
          "decimals": 2,
          "displayName": "",
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Time"
              },
              {
                "id": "custom.align"
              },
              {
                "id": "custom.minWidth",
                "value": 120
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "container"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Container Name"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              },
              {
                "id": "custom.minWidth",
                "value": 150
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #A"
            },
            "properties": [
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #B"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "VPA Mode"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              },
              {
                "id": "mappings",
                "value": [
                  {
                    "options": {
                      "0": {
                        "text": "Off"
                      },
                      "1": {
                        "text": "Initial"
                      },
                      "2": {
                        "text": "Auto"
                      },
                      "3": {
                        "text": "Recreate"
                      },
                      "5": {
                        "text": "-"
                      }
                    },
                    "type": "value"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #C"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "CPU"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #D"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Req CPU"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #E"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "VPA CPU"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #F"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Over-req CPU"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #G"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Under-req CPU"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #H"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Throttling cores"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #I"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Memory"
              },
              {
                "id": "unit",
                "value": "bytes"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #J"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Req Memory"
              },
              {
                "id": "unit",
                "value": "bytes"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #K"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "VPA Memory"
              },
              {
                "id": "unit",
                "value": "bytes"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #L"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Over-req Memory"
              },
              {
                "id": "unit",
                "value": "bytes"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #M"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Under-req Memory"
              },
              {
                "id": "unit",
                "value": "bytes"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #N"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "RX Network (Pod)"
              },
              {
                "id": "unit",
                "value": "Bps"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              },
              {
                "id": "mappings",
                "value": [
                  {
                    "options": {
                      "-1": {
                        "index": 0,
                        "text": "hostNet"
                      }
                    },
                    "type": "value"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #O"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "TX Network (Pod)"
              },
              {
                "id": "unit",
                "value": "Bps"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              },
              {
                "id": "mappings",
                "value": [
                  {
                    "options": {
                      "-1": {
                        "index": 0,
                        "text": "hostNet"
                      }
                    },
                    "type": "value"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #P"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Read IOPS"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #Q"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Write IOPS"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #R"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Pod Restarts"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "custom.align"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "node"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "Node"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "decimals",
                "value": 2
              },
              {
                "id": "custom.align"
              },
              {
                "id": "custom.minWidth",
                "value": 200
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #T"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "OOM killed"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "custom.align"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 141,
      "links": [],
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true
      },
      "pluginVersion": "10.2.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "editorMode": "code",
          "expr": "kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "Container Name",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "max by (pod, node) (avg_over_time(kube_pod_info{namespace=\"$namespace\", pod=\"$pod\"}[$__range]))",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Node",
          "refId": "S"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (pod) (avg_over_time(kube_controller_pod{namespace=\"$namespace\", pod=\"$pod\"}[$__range]) * on (controller_type, controller_name) group_left() sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", update_mode=\"Off\"}[$__range]))) * 0\nor\nsum by (pod) (avg_over_time(kube_controller_pod{namespace=\"$namespace\", pod=\"$pod\"}[$__range]) * on (controller_type, controller_name) group_left() sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", update_mode=\"Initial\"}[$__range]))) * 0 + 1\nor\nsum by (pod) (avg_over_time(kube_controller_pod{namespace=\"$namespace\", pod=\"$pod\"}[$__range]) * on (controller_type, controller_name) group_left() sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", update_mode=\"Auto\"}[$__range]))) * 0 + 2\nor\nsum by (pod) (avg_over_time(kube_controller_pod{namespace=\"$namespace\", pod=\"$pod\"}[$__range]) * on (controller_type, controller_name) group_left() sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", update_mode=\"Recreate\"}[$__range]))) * 0 + 3\nor\nsum by (pod) (avg_over_time(kube_controller_pod{namespace=\"$namespace\", pod=\"$pod\"}[$__range])) * 0 + 5",
          "format": "table",
          "hide": false,
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "VPA Mode",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "hide": false,
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "CPU",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Req CPU",
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (container) (\n  sum by (controller_type, controller_name, pod) (avg_over_time(kube_controller_pod{namespace=\"$namespace\", pod=\"$pod\"}[$__range]))\n  * on (controller_type, controller_name) group_right() \n  sum by (controller_type, controller_name, container) (\n    avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", container=~\"$container\", resource=\"cpu\"}[$__range])))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "VPA CPU",
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "(\n  sum by (container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n  -\n  sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n) > 0\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Over-req CPU",
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "(\n  (\n    sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n    -\n    sum by (container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n  ) or sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n) > 0\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Under-req CPU",
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "(\n  (\n    sum by (container) (rate(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range])) \n    / \n    sum by (container) (rate(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n  )\n  * sum by (container) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n)\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Thottling cores",
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Memory",
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Req Memory",
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (container) (\n  sum by (controller_type, controller_name, pod) (avg_over_time(kube_controller_pod{namespace=\"$namespace\", pod=\"$pod\"}[$__range]))\n  * on (controller_type, controller_name) group_right() \n  sum by (controller_type, controller_name, container) (\n    avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", container=~\"$container\", resource=\"memory\"}[$__range])))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "VPA Memory",
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "(\n  sum by (container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n  -\n  sum by (container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n) > 0\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Over-req Memory",
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "(\n  (\n    sum by (container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n    -\n    sum by (container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n  ) or sum by (container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]))\n) > 0\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Under-req Memory",
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "editorMode": "code",
          "expr": "(\n  # Show data rate for the Pod if it had hostNetwork: false during the selected period.\n  sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"$namespace\", pod=\"$pod\"}[$__range]))\n  * on(pod)\n  max(max_over_time(kube_pod_info{host_network=\"false\", namespace=\"$namespace\", pod=\"$pod\"}[$__range])) by(pod)\n) or (\n  # Else return -1 if the Pod had hostNetwork: true during the selecte period.\n  max_over_time(kube_pod_info{host_network=\"true\", namespace=\"$namespace\", pod=\"$pod\"}[$__range]) * -1\n)",
          "format": "table",
          "hide": false,
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "RX Network",
          "refId": "N"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "editorMode": "code",
          "expr": "(\n  # Show data rate for the Pod if it had hostNetwork: false during the selected period.\n  max(max_over_time(kube_pod_info{host_network=\"false\", namespace=\"$namespace\", pod=\"$pod\"}[$__range])) by(pod)\n  * on(pod)\n  sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"$namespace\", pod=\"$pod\"}[$__range]))\n) or (\n  # Else return -1 if the Pod had hostNetwork: true during the selecte period.\n  max_over_time(kube_pod_info{host_network=\"true\", namespace=\"$namespace\", pod=\"$pod\"}[$__range]) * -1\n)",
          "format": "table",
          "hide": false,
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "TX Network",
          "refId": "O"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by(container) (rate(container_fs_reads_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__range]))",
          "format": "table",
          "hide": false,
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Read IOPS",
          "refId": "P"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by(container) (rate(container_fs_writes_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__range]))",
          "format": "table",
          "hide": false,
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Write IOPS",
          "refId": "Q"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (container) (increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\", pod=\"$pod\"}[$__range]))",
          "format": "table",
          "hide": false,
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Pod Restart count",
          "refId": "R"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (container) (label_replace(increase(klog_pod_oomkill{namespace=\"$namespace\", pod_name=\"$pod\"}[$__range]), \"container\", \"$1\", \"container_name\", \"(.+)\"))",
          "format": "table",
          "hide": false,
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "OOM killed",
          "refId": "T"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${ds_prometheus}"
          },
          "expr": "sum by (container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
          "format": "table",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "Memory",
          "refId": "U"
        }
      ],
      "title": "Containers",
      "transformations": [
        {
          "id": "merge",
          "options": {
            "reducers": []
          }
        },
        {
          "id": "filterFieldsByName",
          "options": {
            "include": {
              "names": [
                "container",
                "node",
                "Value #B",
                "Value #C",
                "Value #D",
                "Value #E",
                "Value #F",
                "Value #G",
                "Value #H",
                "Value #I",
                "Value #J",
                "Value #K",
                "Value #L",
                "Value #M",
                "Value #N",
                "Value #O",
                "Value #P",
                "Value #Q",
                "Value #R"
              ]
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "container"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "fillOpacity": 70,
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineWidth": 0,
            "spanNulls": false
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "color": "#9ac48a",
                  "index": 0,
                  "text": "ready"
                },
                "1": {
                  "color": "#f4d598",
                  "index": 1,
                  "text": "running"
                },
                "2": {
                  "color": "#890f02",
                  "index": 2,
                  "text": "terminated/Error"
                },
                "3": {
                  "color": "#e24d42",
                  "index": 3,
                  "text": "terminated/Completed"
                },
                "4": {
                  "color": "#f29191",
                  "index": 4,
                  "text": "terminated/ContainerCannotRun"
                },
                "5": {
                  "color": "#fce2de",
                  "index": 5,
                  "text": "terminated/OOMKilled"
                },
                "6": {
                  "color": "#1f78c1",
                  "index": 6,
                  "text": "waiting/ContainerCreating"
                },
                "7": {
                  "color": "#64b0c8",
                  "index": 7,
                  "text": "waiting/CrashLoopBackoff"
                },
                "8": {
                  "color": "#65c5db",
                  "index": 8,
                  "text": "waiting/CreateContainerConfigError"
                },
                "9": {
                  "color": "#badff4",
                  "index": 9,
                  "text": "waiting/ErrImagePull"
                },
                "10": {
                  "color": "#cffaff",
                  "index": 10,
                  "text": "waiting/ImagePullBackOff"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 47,
      "links": [],
      "options": {
        "alignValue": "left",
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "mergeValues": false,
        "rowHeight": 0.9,
        "showValue": "never",
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "min by (container) (\n  ((min_over_time(kube_pod_container_status_ready{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) == 1) * 0) or\n  ((min_over_time(kube_pod_container_status_running{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) == 1) * 1) or\n  (\n    (max_over_time(kube_pod_container_status_terminated_reason{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) == 1)\n    * on(reason) group_left()\n    (\n      label_replace(vector(2), \"reason\", \"Error\", \"\", \"\") or\n      label_replace(vector(3), \"reason\", \"Completed\", \"\", \"\") or\n      label_replace(vector(4), \"reason\", \"ContainerCannotRun\", \"\", \"\") or\n      label_replace(vector(5), \"reason\", \"OOMKilled\", \"\", \"\")\n    )\n  ) or\n  (\n    (max_over_time(kube_pod_container_status_waiting_reason{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) == 1)\n    * on(reason) group_left()\n    (\n      label_replace(vector(6), \"reason\", \"ContainerCreating\", \"\", \"\") or\n      label_replace(vector(7), \"reason\", \"CrashLoopBackOff\", \"\", \"\") or\n      label_replace(vector(8), \"reason\", \"CreateContainerConfigError\", \"\", \"\") or\n      label_replace(vector(9), \"reason\", \"ErrImagePull\", \"\", \"\") or\n      label_replace(vector(10), \"reason\", \"ImagePullBackOff\", \"\", \"\")\n    )\n  )\n)",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "A"
        }
      ],
      "title": "Status",
      "type": "state-timeline"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "The number of container restarts",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 4,
        "w": 12,
        "x": 0,
        "y": 17
      },
      "id": 166,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by (container) (increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "A"
        }
      ],
      "title": "Containers restarts",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "The number of container OOM kills",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 4,
        "w": 12,
        "x": 12,
        "y": 17
      },
      "id": 167,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by (container) (increase(oom_kills:normalized{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "A"
        }
      ],
      "title": "OOM Killed Processes",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 21
      },
      "id": 14,
      "panels": [],
      "title": "CPU",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "cores",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 22
      },
      "id": 2,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true,
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
          "format": "time_series",
          "instant": false,
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "A"
        }
      ],
      "title": "Usage by container",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "The total value may not equal the sum of system and user times because of the kernel's cgroup accounting peculiarities. Read more here: https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "cores",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "System"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#e24d42",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "User"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#1f78c1",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": "A",
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 22
      },
      "id": 3,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true,
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by(pod) (rate(container_cpu_system_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
          "format": "time_series",
          "instant": false,
          "intervalFactor": 1,
          "legendFormat": "System",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by(pod) (rate(container_cpu_user_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "User",
          "refId": "B"
        }
      ],
      "title": "Usage by state",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "This graph shows the requested CPU resources higher than the actual CPU consumption. In other words, it shows CPU resources that can be \"freed\" without affecting the service.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "cores",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 31
      },
      "id": 76,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true,
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by (namespace, pod, container)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{container!=\"\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n    ) > 0\n  )",
          "format": "time_series",
          "hide": false,
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "C"
        }
      ],
      "title": "Over-requested by container",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "This graph shows the requested CPU resources lower than the actual CPU consumption. In other words, it shows CPU resources that need to be \"reserved\" for the service to run smoothly.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "cores",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 31
      },
      "id": 123,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by (namespace, pod, container)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n    ) > 0\n  )",
          "format": "time_series",
          "hide": false,
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "C"
        }
      ],
      "title": "Under-requested by container",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "The absence of data on the graph means that container resources are not set",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "cores",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 6,
        "w": 24,
        "x": 0,
        "y": 39
      },
      "id": 36,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "max"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "(sum by (pod, container) (rate(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", container=~\"$container\", pod=\"$pod\"}[$__rate_interval])) / sum by (container, pod) (rate(container_cpu_cfs_periods_total{namespace=\"$namespace\", container=~\"$container\", pod=\"$pod\"}[$__rate_interval]))) * sum by (pod, container) (kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "A"
        }
      ],
      "title": "Throttling",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 45
      },
      "id": 95,
      "panels": [
        {
          "datasource": {
            "uid": "$ds_prometheus"
          },
          "description": "The total value may not equal the sum of system and user times because of the kernel's cgroup accounting peculiarities. Read more here: https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "cores",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "smooth",
                "lineWidth": 2,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Usage"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#629e51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Requests"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#f4d598",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Limits"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#c15c17",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "VPA Target"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#447ebc",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 46
          },
          "id": 102,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "multi",
              "sort": "desc"
            }
          },
          "pluginVersion": "8.5.13",
          "repeat": "container",
          "repeatDirection": "h",
          "targets": [
            {
              "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Usage",
              "refId": "D"
            },
            {
              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=\"$container\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Requests",
              "refId": "C"
            },
            {
              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=\"$container\"}[$__rate_interval]) )",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Limits",
              "refId": "E"
            },
            {
              "expr": "sum by (container) (\n  sum by (controller_type, controller_name, pod) (kube_controller_pod{namespace=\"$namespace\", pod=\"$pod\"})\n  * on (controller_type, controller_name) group_right() \n  sum by (controller_type, controller_name, container) (\n    avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", container=~\"$container\", resource=\"cpu\"}[$__rate_interval])\n  )\n)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "VPA Target",
              "refId": "F"
            }
          ],
          "title": "$container",
          "type": "timeseries"
        }
      ],
      "title": "Containers CPU",
      "type": "row"
    },
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 46
      },
      "id": 5,
      "panels": [],
      "title": "Containers CPU by state",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "The total value may not equal the sum of system and user times because of the kernel's cgroup accounting peculiarities. Read more here: https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "cores",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "System"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#e24d42",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "User"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#1f78c1",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 0,
        "y": 47
      },
      "id": 7,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true,
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "repeat": "container",
      "repeatDirection": "h",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by(container) (rate(container_cpu_system_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
          "format": "time_series",
          "instant": false,
          "intervalFactor": 1,
          "legendFormat": "System",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by(container) (rate(container_cpu_user_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "User",
          "refId": "B"
        }
      ],
      "title": "$container",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 65
      },
      "id": 12,
      "panels": [],
      "title": "Memory",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 66
      },
      "id": 6,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
          "format": "time_series",
          "instant": false,
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "A"
        }
      ],
      "title": "Usage by container",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "$ds_prometheus"
      },
      "description": "The Working set bytes metric is the actual memory used by the container, as it includes active file memory. When its value approaches the limit, the container can be killed by the OOMKiller. This value can be higher than the sum RSS and Cache since not all active file memory is Cache.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Working set bytes without kmem"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "rgb(0, 0, 0)",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "custom.lineWidth",
                "value": 2
              },
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineStyle",
                "value": {
                  "dash": [
                    10,
                    10
                  ],
                  "fill": "dash"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Kmem"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "rgb(255, 0, 0)",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "custom.lineWidth",
                "value": 2
              },
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineStyle",
                "value": {
                  "dash": [
                    10,
                    10
                  ],
                  "fill": "dash"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 66
      },
      "id": 8,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "expr": "sum by(pod) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
          "format": "time_series",
          "instant": false,
          "intervalFactor": 1,
          "legendFormat": "RSS",
          "refId": "A"
        },
        {
          "expr": "sum by(pod) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Cache",
          "refId": "B"
        },
        {
          "expr": "sum by(pod) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Swap",
          "refId": "C"
        },
        {
          "expr": "sum by(pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Working set bytes without kmem",
          "refId": "D"
        },
        {
          "expr": "sum by(pod) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Kmem",
          "refId": "E"
        }
      ],
      "title": "Usage by state",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "This graph shows the requested Memory resources higher than the actual Memory consumption. In other words, it shows Memory resources that can be \"freed\" without affecting the service.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 75
      },
      "id": 106,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by (container)\n  (\n    (\n      sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"\"}[$__rate_interval]))\n    ) > 0\n  )",
          "format": "time_series",
          "hide": false,
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "C"
        }
      ],
      "title": "Over-requested by container",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "This graph shows the requested Memory resources higher than the actual Memory consumption. In other words, it shows Memory resources that need to be \"reserved\" for the service to run smoothly.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 75
      },
      "id": 136,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by (container)\n  (\n    (\n      (\n        sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n        -\n        sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"\"}[$__rate_interval]))\n      ) or sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"\"}[$__rate_interval]))\n    ) > 0\n  )",
          "format": "time_series",
          "hide": false,
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "C"
        }
      ],
      "title": "Under-requested by container",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 83
      },
      "id": 16,
      "panels": [
        {
          "datasource": {
            "uid": "$ds_prometheus"
          },
          "description": "The Working set bytes metric is the actual memory used by the container, as it includes active file memory. When its value approaches the limit, the container can be killed by the OOMKiller. This value can be higher than the sum RSS and Cache since not all active file memory is Cache.",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "smooth",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Working set bytes without kmem"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "rgb(0, 0, 0)",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.fillOpacity",
                    "value": 0
                  },
                  {
                    "id": "custom.lineWidth",
                    "value": 2
                  },
                  {
                    "id": "custom.stacking",
                    "value": {
                      "group": false,
                      "mode": "none"
                    }
                  },
                  {
                    "id": "custom.lineStyle",
                    "value": {
                      "dash": [
                        10,
                        10
                      ],
                      "fill": "dash"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Kmem"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "rgb(255, 0, 0)",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.fillOpacity",
                    "value": 0
                  },
                  {
                    "id": "custom.lineWidth",
                    "value": 2
                  },
                  {
                    "id": "custom.stacking",
                    "value": {
                      "group": false,
                      "mode": "normal"
                    }
                  },
                  {
                    "id": "custom.lineStyle",
                    "value": {
                      "dash": [
                        10,
                        10
                      ],
                      "fill": "dash"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Limits"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#c15c17",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.fillOpacity",
                    "value": 0
                  },
                  {
                    "id": "custom.lineWidth",
                    "value": 0
                  },
                  {
                    "id": "custom.lineWidth",
                    "value": 2
                  },
                  {
                    "id": "custom.stacking",
                    "value": {
                      "group": false,
                      "mode": "none"
                    }
                  },
                  {
                    "id": "custom.lineStyle",
                    "value": {
                      "dash": [
                        10,
                        10
                      ],
                      "fill": "dash"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Requests"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#f4d598",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.fillOpacity",
                    "value": 0
                  },
                  {
                    "id": "custom.lineWidth",
                    "value": 0
                  },
                  {
                    "id": "custom.lineWidth",
                    "value": 2
                  },
                  {
                    "id": "custom.stacking",
                    "value": {
                      "group": false,
                      "mode": "none"
                    }
                  },
                  {
                    "id": "custom.lineStyle",
                    "value": {
                      "dash": [
                        10,
                        10
                      ],
                      "fill": "dash"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "VPA Target"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#447ebc",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.fillOpacity",
                    "value": 0
                  },
                  {
                    "id": "custom.lineWidth",
                    "value": 0
                  },
                  {
                    "id": "custom.lineWidth",
                    "value": 2
                  },
                  {
                    "id": "custom.stacking",
                    "value": {
                      "group": false,
                      "mode": "none"
                    }
                  },
                  {
                    "id": "custom.lineStyle",
                    "value": {
                      "dash": [
                        10,
                        10
                      ],
                      "fill": "dash"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 75
          },
          "id": 19,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": false
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "repeat": "container",
          "repeatDirection": "h",
          "targets": [
            {
              "expr": "sum by(container) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "RSS",
              "refId": "A"
            },
            {
              "expr": "sum by(container) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Cache",
              "refId": "B"
            },
            {
              "expr": "sum by(container) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Swap",
              "refId": "C"
            },
            {
              "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Working set bytes without kmem",
              "refId": "D"
            },
            {
              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Limits",
              "refId": "E"
            },
            {
              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Requests",
              "refId": "F"
            },
            {
              "expr": "sum by (container) (\n  sum by (controller_type, controller_name, pod) (kube_controller_pod{namespace=\"$namespace\", pod=\"$pod\"})\n  * on (controller_type, controller_name) group_right() \n  sum by (controller_type, controller_name, container) (\n    avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", container=~\"$container\", resource=\"memory\"}[$__rate_interval])\n  )\n)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "VPA Target",
              "refId": "G"
            },
            {
              "expr": "sum by(container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Kmem",
              "refId": "H"
            }
          ],
          "title": "$container",
          "type": "timeseries"
        }
      ],
      "title": "Containers Memory",
      "type": "row"
    },
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 84
      },
      "id": 25,
      "panels": [],
      "title": "Network",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "This graph shows Network Receive (except for the hostNetwork Pods)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [
            {
              "options": {
                "-1": {
                  "color": "transparent",
                  "index": 0,
                  "text": "Network graph for Pod with hostNetwork: true is meaningless"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          },
          "unit": "Bps"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byFrameRefID",
              "options": "B"
            },
            "properties": [
              {
                "id": "min",
                "value": 0
              },
              {
                "id": "max",
                "value": 0
              },
              {
                "id": "custom.axisPlacement",
                "value": "hidden"
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "transparent",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 85
      },
      "id": 27,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "expr": "# Select Pods with hostNetwork: false.\nkube_pod_info{host_network=\"false\",namespace=\"$namespace\", pod=\"$pod\"}\n* on(pod)\n# Sum data rate for all interfaces in the Pod.\nsum by(pod) (rate(container_network_receive_bytes_total{pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{ pod }}",
          "refId": "A"
        },
        {
          "expr": "# Return -1 for Pods with 'hostNetwork: true' to use in value mappings.\nkube_pod_info{host_network=\"true\",namespace=\"$namespace\", pod=\"$pod\"}\n*\n-1",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{pod}}",
          "refId": "B"
        }
      ],
      "title": "Receive",
      "transformations": [],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "This graph shows Network Transmit (except for the hostNetwork Pods)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [
            {
              "options": {
                "-1": {
                  "color": "transparent",
                  "index": 0,
                  "text": "no traffic available for hostNetwork: true"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          },
          "unit": "Bps"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byFrameRefID",
              "options": "B"
            },
            "properties": [
              {
                "id": "min",
                "value": 0
              },
              {
                "id": "max",
                "value": 0
              },
              {
                "id": "custom.axisPlacement",
                "value": "hidden"
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "transparent",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 85
      },
      "id": 62,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true,
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "expr": "# Select Pods with hostNetwork: false.\nkube_pod_info{host_network=\"false\",namespace=\"$namespace\", pod=\"$pod\"}\n* on(pod)\n# Sum data rate for all interfaces in the Pod.\nsum by(pod) (rate(container_network_transmit_bytes_total{pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{ pod }}",
          "refId": "A"
        },
        {
          "expr": "# Return -1 for Pods with 'hostNetwork: true' to use in value mappings.\nkube_pod_info{host_network=\"true\",namespace=\"$namespace\", pod=\"$pod\"}\n*\n-1",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{pod}}",
          "refId": "B"
        }
      ],
      "title": "Transmit",
      "transformations": [],
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 92
      },
      "id": 30,
      "panels": [],
      "title": "IOPS",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "iops"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 93
      },
      "id": 32,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true,
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "A"
        }
      ],
      "title": "Read",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "iops"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Total"
            },
            "properties": [
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 93
      },
      "id": 63,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true,
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "sum by(container) (rate(container_fs_writes_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{ container }}",
          "refId": "A"
        }
      ],
      "title": "Write",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 101
      },
      "id": 150,
      "panels": [],
      "title": "PVC",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "This graph does not show any localstorage-related information due to incorrectly calculating the occupied space on localstorage disks",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/Total/"
            },
            "properties": [
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 102
      },
      "id": 148,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__rate_interval])\n* \non (namespace, persistentvolumeclaim) group_left (pod) \n(\n  avg_over_time(kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])\n  * on (namespace, persistentvolumeclaim) group_right(pod) \n  max by (namespace, persistentvolumeclaim) (\n    label_replace(\n      max_over_time(kube_persistentvolumeclaim_info{namespace=\"$namespace\"}[$__rate_interval]),\n      \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n    + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0))\n  * on (namespace, persistentvolumeclaim) group_right (pod)\n  avg_over_time(kube_persistentvolumeclaim_status_phase{namespace=\"$namespace\", phase=\"Bound\"}[$__rate_interval])\n)\n",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Provisioned {{ persistentvolumeclaim }}",
          "refId": "A"
        }
      ],
      "title": "PVC Provisioned (except local storage classes)",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "description": "This graph does not show any localstorage-related information due to incorrectly calculating the occupied space on localstorage disks",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/Total/"
            },
            "properties": [
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "custom.lineWidth",
                "value": 0
              },
              {
                "id": "custom.stacking",
                "value": {
                  "group": false,
                  "mode": "none"
                }
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 102
      },
      "id": 151,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "(avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__rate_interval]) - avg_over_time(kubelet_volume_stats_available_bytes{namespace=\"$namespace\"}[$__rate_interval]))\n* \non (namespace, persistentvolumeclaim) group_left (pod) \n(\n  avg_over_time(kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])\n  * on (namespace, persistentvolumeclaim) group_right(pod) \n  max by (namespace, persistentvolumeclaim) (\n    label_replace(\n      max_over_time(kube_persistentvolumeclaim_info{namespace=\"$namespace\"}[$__rate_interval]),\n      \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n    + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0))\n  * on (namespace, persistentvolumeclaim) group_right (pod)\n  avg_over_time(kube_persistentvolumeclaim_status_phase{namespace=\"$namespace\", phase=\"Bound\"}[$__rate_interval])\n)",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Usage {{ persistentvolumeclaim }}",
          "refId": "A"
        }
      ],
      "title": "PVC Usage (except local storage classes)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "uid": "$ds_prometheus"
      },
      "description": "This graph does not show any localstorage-related information due to incorrectly calculating the occupied space on localstorage disks",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 110
      },
      "id": 152,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "expr": "(avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__rate_interval]) - avg_over_time(kubelet_volume_stats_available_bytes{namespace=\"$namespace\"}[$__rate_interval]))\n* \non (namespace, persistentvolumeclaim) group_left (pod) \n(\n  avg_over_time(kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])\n  * on (namespace, persistentvolumeclaim) group_right(pod) \n  max by (namespace, persistentvolumeclaim) (\n    label_replace(\n      max_over_time(kube_persistentvolumeclaim_info{namespace=\"$namespace\"}[$__rate_interval]),\n      \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n    + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0))\n  * on (namespace, persistentvolumeclaim) group_right (pod)\n  avg_over_time(kube_persistentvolumeclaim_status_phase{namespace=\"$namespace\", phase=\"Bound\"}[$__rate_interval])\n)\n/\n(\n  avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__rate_interval])\n  * \n  on (namespace, persistentvolumeclaim) group_left (pod) \n  (\n    avg_over_time(kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])\n    * on (namespace, persistentvolumeclaim) group_right(pod) \n    max by (namespace, persistentvolumeclaim) (\n      label_replace(\n        max_over_time(kube_persistentvolumeclaim_info{namespace=\"$namespace\"}[$__rate_interval]),\n        \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\")\n      + on (persistentvolume) group_left() max by (persistentvolume) (kube_persistentvolume_is_local == 0))\n    * on (namespace, persistentvolumeclaim) group_right (pod)\n    avg_over_time(kube_persistentvolumeclaim_status_phase{namespace=\"$namespace\", phase=\"Bound\"}[$__rate_interval])\n  )\n)",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Usage {{ persistentvolumeclaim }}",
          "refId": "B"
        }
      ],
      "title": "PVC Usage in % (except local storage classes)",
      "type": "timeseries"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 118
      },
      "id": 156,
      "panels": [
        {
          "datasource": {
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "cellOptions": {
                  "type": "auto"
                },
                "inspect": false
              },
              "decimals": 2,
              "displayName": "",
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Time"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "Time"
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "persistentvolumeclaim"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "Name"
                  },
                  {
                    "id": "unit",
                    "value": "short"
                  },
                  {
                    "id": "decimals",
                    "value": 2
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "storageclass"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "StorageClass"
                  },
                  {
                    "id": "unit",
                    "value": "short"
                  },
                  {
                    "id": "decimals",
                    "value": 2
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Value #B"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "Requested"
                  },
                  {
                    "id": "unit",
                    "value": "bytes"
                  },
                  {
                    "id": "decimals",
                    "value": 2
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Value #C"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "Provisioned"
                  },
                  {
                    "id": "unit",
                    "value": "bytes"
                  },
                  {
                    "id": "decimals",
                    "value": 2
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Value #D"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "Capacity"
                  },
                  {
                    "id": "unit",
                    "value": "bytes"
                  },
                  {
                    "id": "decimals",
                    "value": 2
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Value #E"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "Used bytes"
                  },
                  {
                    "id": "unit",
                    "value": "bytes"
                  },
                  {
                    "id": "decimals",
                    "value": 2
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Value #F"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "Used inodes"
                  },
                  {
                    "id": "unit",
                    "value": "short"
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Value #G"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "Used bytes (%)"
                  },
                  {
                    "id": "unit",
                    "value": "percentunit"
                  },
                  {
                    "id": "decimals",
                    "value": 2
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Value #H"
                },
                "properties": [
                  {
                    "id": "displayName",
                    "value": "Used inodes (%)"
                  },
                  {
                    "id": "unit",
                    "value": "percentunit"
                  },
                  {
                    "id": "decimals",
                    "value": 2
                  },
                  {
                    "id": "custom.align"
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 110
          },
          "id": 154,
          "links": [],
          "options": {
            "cellHeight": "sm",
            "footer": {
              "countRows": false,
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true
          },
          "pluginVersion": "10.2.2",
          "targets": [
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  kube_persistentvolumeclaim_info{namespace=\"$namespace\"}\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "Name, storageclass",
              "refId": "A"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace=\"$namespace\"}[$__range])\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "Requested (pvc)",
              "refId": "B"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  label_replace(\n    kube_persistentvolumeclaim_info{namespace=\"$namespace\"},\n    \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\") \n  * on (persistentvolume) group_left() \n  kube_persistentvolume_capacity_bytes\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "table",
              "hide": false,
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "Provisioned (pv)",
              "refId": "C"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__range])\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "Capcity (real)",
              "refId": "D"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__range]) - avg_over_time(kubelet_volume_stats_available_bytes{namespace=\"$namespace\"}[$__range])\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "Used bytes",
              "refId": "E"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  avg_over_time(kubelet_volume_stats_inodes_used{namespace=\"$namespace\"}[$__range])\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "Used inodes",
              "refId": "F"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  (avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__range]) - avg_over_time(kubelet_volume_stats_available_bytes{namespace=\"$namespace\"}[$__range]))\n  /\n  avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__range])\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "Used bytes %",
              "refId": "G"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  avg_over_time(kubelet_volume_stats_inodes_used{namespace=\"$namespace\"}[$__range])\n  /\n  avg_over_time(kubelet_volume_stats_inodes{namespace=\"$namespace\"}[$__range])\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "Used inodes %",
              "refId": "H"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, pod) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "pod",
              "refId": "I"
            }
          ],
          "title": "Overview",
          "transformations": [
            {
              "id": "merge",
              "options": {
                "reducers": []
              }
            },
            {
              "id": "filterFieldsByName",
              "options": {
                "include": {
                  "names": [
                    "persistentvolumeclaim",
                    "storageclass",
                    "Value #B",
                    "Value #C",
                    "Value #D",
                    "Value #E",
                    "Value #F",
                    "Value #G",
                    "Value #H"
                  ]
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "datasource": {
            "uid": "$ds_prometheus"
          },
          "description": "This graph does not show any localstorage-related information due to incorrectly calculating the occupied space on localstorage disks",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisBorderShow": false,
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "insertNulls": false,
                "lineInterpolation": "smooth",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 119
          },
          "id": 157,
          "links": [],
          "options": {
            "legend": {
              "calcs": [
                "mean"
              ],
              "displayMode": "table",
              "placement": "right",
              "showLegend": true
            },
            "tooltip": {
              "mode": "multi",
              "sort": "desc"
            }
          },
          "pluginVersion": "8.5.13",
          "repeat": "persistentvolumeclaim",
          "repeatDirection": "h",
          "targets": [
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  label_replace(\n    kube_persistentvolumeclaim_info{namespace=\"$namespace\"},\n    \"persistentvolume\", \"$1\", \"volumename\", \"(.*)\") \n  * on (persistentvolume) group_left() \n  avg_over_time(kube_persistentvolume_capacity_bytes[$__rate_interval])\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\", pod=~\"$pod\"}\n)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Provisioned",
              "refId": "A"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__rate_interval])\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\", pod=~\"$pod\"}\n)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Capacity",
              "refId": "B"
            },
            {
              "expr": "max by (namespace, persistentvolumeclaim, storageclass) (\n  avg_over_time(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}[$__rate_interval]) - avg_over_time(kubelet_volume_stats_available_bytes{namespace=\"$namespace\"}[$__rate_interval])\n) \nand on (namespace, persistentvolumeclaim) \nmax by (namespace, persistentvolumeclaim) (\n  kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=~\"$pod\"}\n)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Used",
              "refId": "C"
            }
          ],
          "title": "$persistentvolumeclaim",
          "type": "timeseries"
        }
      ],
      "title": "PVC Detailed",
      "type": "row"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 38,
  "tags": [
    "main"
  ],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "default",
          "value": "default"
        },
        "hide": 0,
        "includeAll": false,
        "label": "Prometheus",
        "multi": false,
        "name": "ds_prometheus",
        "options": [],
        "query": "prometheus",
        "queryValue": "",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "current": {
          "selected": false,
          "text": "d8-monitoring",
          "value": "d8-monitoring"
        },
        "datasource": {
          "type": "prometheus",
          "uid": "$ds_prometheus"
        },
        "definition": "",
        "hide": 0,
        "includeAll": false,
        "label": "Namespace",
        "multi": false,
        "name": "namespace",
        "options": [],
        "query": {
          "query": "label_values(kube_pod_info, namespace)",
          "refId": "main-namespace-Variable-Query"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {
          "selected": false,
          "text": "prometheus-main-0",
          "value": "prometheus-main-0"
        },
        "datasource": {
          "type": "prometheus",
          "uid": "$ds_prometheus"
        },
        "definition": "",
        "hide": 0,
        "includeAll": false,
        "label": "Pod",
        "multi": false,
        "name": "pod",
        "options": [],
        "query": {
          "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)",
          "refId": "main-pod-Variable-Query"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": "",
        "current": {
          "selected": false,
          "text": [
            "All"
          ],
          "value": [
            "$__all"
          ]
        },
        "datasource": {
          "type": "prometheus",
          "uid": "$ds_prometheus"
        },
        "definition": "",
        "hide": 2,
        "includeAll": true,
        "label": "Container",
        "multi": true,
        "name": "container",
        "options": [],
        "query": {
          "query": "label_values(kube_pod_container_info{namespace=~\"$namespace\", pod=~\"$pod\"}, container)",
          "refId": "main-container-Variable-Query"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": ".*",
        "current": {
          "selected": false,
          "text": [
            "All"
          ],
          "value": [
            "$__all"
          ]
        },
        "datasource": {
          "type": "prometheus",
          "uid": "$ds_prometheus"
        },
        "definition": "label_values(kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=\"$pod\"}, persistentvolumeclaim)",
        "hide": 2,
        "includeAll": true,
        "label": "PersistentVolumeClaim",
        "multi": true,
        "name": "persistentvolumeclaim",
        "options": [],
        "query": {
          "query": "label_values(kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=\"$namespace\", pod=\"$pod\"}, persistentvolumeclaim)",
          "refId": "main-persistentvolumeclaim-Variable-Query"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {
          "selected": false,
          "text": "Deployment",
          "value": "Deployment"
        },
        "datasource": {
          "type": "prometheus",
          "uid": "$ds_prometheus"
        },
        "definition": "label_values(kube_controller_pod{namespace=~\"$namespace\"}, controller_type)",
        "hide": 2,
        "includeAll": false,
        "label": "Controller Type",
        "multi": false,
        "name": "controller_type",
        "options": [],
        "query": {
          "query": "label_values(kube_controller_pod{namespace=~\"$namespace\"}, controller_type)",
          "refId": "main-controller_type-Variable-Query"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {
          "selected": false,
          "text": "aggregating-proxy",
          "value": "aggregating-proxy"
        },
        "datasource": {
          "type": "prometheus",
          "uid": "$ds_prometheus"
        },
        "definition": "label_values(kube_controller_pod{namespace=~\"$namespace\"}, controller_name)",
        "hide": 2,
        "includeAll": false,
        "label": "Controller Name",
        "multi": false,
        "name": "controller_name",
        "options": [],
        "query": {
          "query": "label_values(kube_controller_pod{namespace=~\"$namespace\"}, controller_name)",
          "refId": "main-controller_name-Variable-Query"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$ds_prometheus"
        },
        "filters": [],
        "hide": 0,
        "name": "Filters",
        "skipUrlSync": false,
        "type": "adhoc"
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "Namespace / Controller / Pod",
  "uid": "m41rB4yiz1",
  "version": 2,
  "weekStart": ""
}
