apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: alerts-kubernetes-resources
spec:
  groups:
  - name: kubernetes-resources
    params: {}
    rules:
    - alert: KubeCPUOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }} has overcommitted CPU resource
          requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit
        summary: Cluster has overcommitted CPU resource requests.
      expr: |-
        sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) by (cluster) - (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
        and
        (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
      for: 10m
      labels:
        severity: warning
        exported_instance: '{{ $labels.cluster }}'
        service: kubernetes-resources
    - alert: KubeMemoryOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }} has overcommitted memory resource
          requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node
          failure.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit
        summary: Cluster has overcommitted memory resource requests.
      expr: |-
        sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by (cluster) - (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
        and
        (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
      for: 10m
      labels:
        severity: warning
        exported_instance: '{{ $labels.cluster }}'
        service: kubernetes-resources
    - alert: KubeCPUQuotaOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }}  has overcommitted CPU resource
          requests for Namespaces.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit
        summary: Cluster has overcommitted CPU resource requests.
      expr: |-
        sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) by (cluster)
          /
        sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) by (cluster)
          > 1.5
      for: 5m
      labels:
        severity: warning
        exported_instance: '{{ $labels.cluster }}'
        service: kubernetes-resources
    - alert: KubeMemoryQuotaOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }}  has overcommitted memory resource
          requests for Namespaces.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit
        summary: Cluster has overcommitted memory resource requests.
      expr: |-
        sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) by (cluster)
          /
        sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)
          > 1.5
      for: 5m
      labels:
        severity: warning
        exported_instance: '{{ $labels.cluster }}'
        service: kubernetes-resources
    - alert: KubeQuotaAlmostFull
      annotations:
        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
          }} of its {{ $labels.resource }} quota.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull
        summary: Namespace quota is going to be full.
      expr: |-
        kube_resourcequota{job="kube-state-metrics", type="used"}
          / ignoring(instance, job, type)
        (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
          > 0.9 < 1
      for: 15m
      labels:
        severity: informational
        exported_instance: '{{ $labels.namespace }}'
        service: kubernetes-resources
    - alert: KubeQuotaFullyUsed
      annotations:
        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
          }} of its {{ $labels.resource }} quota.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused
        summary: Namespace quota is fully used.
      expr: |-
        kube_resourcequota{job="kube-state-metrics", type="used"}
          / ignoring(instance, job, type)
        (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
          == 1
      for: 15m
      labels:
        severity: informational
        exported_instance: '{{ $labels.namespace }}'
        service: kubernetes-resources
    - alert: KubeQuotaExceeded
      annotations:
        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
          }} of its {{ $labels.resource }} quota.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded
        summary: Namespace quota has exceeded the limits.
      expr: |-
        kube_resourcequota{job="kube-state-metrics", type="used"}
          / ignoring(instance, job, type)
        (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
          > 1
      for: 15m
      labels:
        severity: warning
        exported_instance: '{{ $labels.namespace }}'
        service: kubernetes-resources
    - alert: CPUThrottlingHigh
      annotations:
        description: '{{ $value | humanizePercentage }} throttling of CPU in namespace
          {{ $labels.namespace }} for container {{ $labels.container }} in pod {{
          $labels.pod }}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh
        summary: Processes experience elevated CPU throttling.
      expr: |-
        sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container,pod,namespace,cluster)
          /
        sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container,pod,namespace,cluster)
          > ( 25 / 100 )
      for: 15m
      labels:
        severity: informational
        exported_instance: '{{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container
          }}'
        service: kubernetes-resources
