apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: alerts-kubernetes-system-kubelet
spec:
  groups:
  - name: kubernetes-system-kubelet
    params: {}
    rules:
    - alert: KubeNodeNotReady
      annotations:
        description: '{{ $labels.node }} has been unready for more than 15 minutes.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready
        summary: Node is not ready.
      expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"}
        == 0
      for: 15m
      labels:
        severity: warning
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeNodeUnreachable
      annotations:
        description: '{{ $labels.node }} is unreachable and some workloads may be
          rescheduled.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeunreachable
        summary: Node is unreachable.
      expr: (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"}
        unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"})
        == 1
      for: 15m
      labels:
        severity: warning
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletTooManyPods
      annotations:
        description: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage
          }} of its Pod capacity.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods
        summary: Kubelet is running at capacity.
      expr: |-
        count by (node,cluster) (
          (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on (instance,pod,namespace,cluster) group_left(node) topk by (instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
        )
        /
        max by (node,cluster) (
          kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1
        ) > 0.95
      for: 15m
      labels:
        severity: informational
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeNodeReadinessFlapping
      annotations:
        description: The readiness status of node {{ $labels.node }} has changed {{
          $value }} times in the last 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping
        summary: Node readiness status is flapping.
      expr: sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready"}[15m]))
        by (node,cluster) > 2
      for: 15m
      labels:
        severity: warning
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletPlegDurationHigh
      annotations:
        description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile
          duration of {{ $value }} seconds on node {{ $labels.node }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletplegdurationhigh
        summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist.
      expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"}
        >= 10
      for: 5m
      labels:
        severity: warning
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletPodStartUpLatencyHigh
      annotations:
        description: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds
          on node {{ $labels.node }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh
        summary: Kubelet Pod startup latency is too high.
      expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet",
        metrics_path="/metrics"}[5m])) by (instance,le,cluster)) * on (instance,cluster)
        group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}
        > 60
      for: 15m
      labels:
        severity: warning
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletClientCertificateExpiration
      annotations:
        description: Client certificate for Kubelet on node {{ $labels.node }} expires
          in {{ $value | humanizeDuration }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration
        summary: Kubelet client certificate is about to expire.
      expr: kubelet_certificate_manager_client_ttl_seconds < 604800
      labels:
        severity: warning
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletClientCertificateExpiration
      annotations:
        description: Client certificate for Kubelet on node {{ $labels.node }} expires
          in {{ $value | humanizeDuration }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration
        summary: Kubelet client certificate is about to expire.
      expr: kubelet_certificate_manager_client_ttl_seconds < 86400
      labels:
        severity: critical
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletServerCertificateExpiration
      annotations:
        description: Server certificate for Kubelet on node {{ $labels.node }} expires
          in {{ $value | humanizeDuration }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration
        summary: Kubelet server certificate is about to expire.
      expr: kubelet_certificate_manager_server_ttl_seconds < 604800
      labels:
        severity: warning
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletServerCertificateExpiration
      annotations:
        description: Server certificate for Kubelet on node {{ $labels.node }} expires
          in {{ $value | humanizeDuration }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration
        summary: Kubelet server certificate is about to expire.
      expr: kubelet_certificate_manager_server_ttl_seconds < 86400
      labels:
        severity: critical
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletClientCertificateRenewalErrors
      annotations:
        description: Kubelet on node {{ $labels.node }} has failed to renew its client
          certificate ({{ $value | humanize }} errors in the last 5 minutes).
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificaterenewalerrors
        summary: Kubelet has failed to renew its client certificate.
      expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m])
        > 0
      for: 15m
      labels:
        severity: warning
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletServerCertificateRenewalErrors
      annotations:
        description: Kubelet on node {{ $labels.node }} has failed to renew its server
          certificate ({{ $value | humanize }} errors in the last 5 minutes).
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificaterenewalerrors
        summary: Kubelet has failed to renew its server certificate.
      expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0
      for: 15m
      labels:
        severity: warning
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
    - alert: KubeletDown
      annotations:
        description: Kubelet has disappeared from Prometheus target discovery.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletdown
        summary: Target disappeared from Prometheus target discovery.
      expr: absent(up{job="kubelet", metrics_path="/metrics"} == 1)
      for: 15m
      labels:
        severity: critical
        exported_instance: '{{ $labels.node }}'
        service: kubernetes-system-kubelet
