apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: alerts-kubernetes-system-apiserver
spec:
  groups:
  - name: kubernetes-system-apiserver
    params: {}
    rules:
    - alert: KubeClientCertificateExpiration
      annotations:
        description: A client certificate used to authenticate to kubernetes apiserver
          is expiring in less than 7.0 days.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
        summary: Client certificate is about to expire.
      expr: apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"}
        > 0 and on (job,cluster) histogram_quantile(0.01, sum by (job,le,cluster)
        (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m])))
        < 604800
      for: 5m
      labels:
        severity: informational
        exported_instance: '{{ $labels.namespace }}/{{ $labels.pod }}'
        service: kubernetes-system-apiserver
    - alert: KubeClientCertificateExpiration
      annotations:
        description: A client certificate used to authenticate to kubernetes apiserver
          is expiring in less than 24.0 hours.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
        summary: Client certificate is about to expire.
      expr: apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"}
        > 0 and on (job,cluster) histogram_quantile(0.01, sum by (job,le,cluster)
        (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m])))
        < 86400
      for: 5m
      labels:
        severity: informational
        exported_instance: '{{ $labels.namespace }}/{{ $labels.pod }}'
        service: kubernetes-system-apiserver
    - alert: KubeAggregatedAPIErrors
      annotations:
        description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
          }} has reported errors. It has appeared unavailable {{ $value | humanize
          }} times averaged over the past 10m.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors
        summary: Kubernetes aggregated API has reported errors.
      expr: sum by (name,namespace,cluster)(increase(aggregator_unavailable_apiservice_total{job="kube-apiserver"}[10m]))
        > 4
      labels:
        severity: warning
        exported_instance: '{{ $labels.name }}/{{ $labels.namespace }}'
        service: kubernetes-system-apiserver
    - alert: KubeAggregatedAPIDown
      annotations:
        description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
          }} has been only {{ $value | humanize }}% available over the last 10m.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown
        summary: Kubernetes aggregated API is down.
      expr: (1 - max by (name,namespace,cluster)(avg_over_time(aggregator_unavailable_apiservice{job="kube-apiserver"}[10m])))
        * 100 < 85
      for: 5m
      labels:
        severity: warning
        exported_instance: '{{ $labels.name }}/{{ $labels.namespace }}'
        service: kubernetes-system-apiserver
    - alert: KubeAPIDown
      annotations:
        description: KubeAPI has disappeared from Prometheus target discovery.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapidown
        summary: Target disappeared from Prometheus target discovery.
      expr: absent(up{job="kube-apiserver"} == 1)
      for: 15m
      labels:
        severity: critical
        exported_instance: '{{ $labels.cluster }}/apiserver'
        service: kubernetes-system-apiserver
    - alert: KubeAPITerminatedRequests
      annotations:
        description: The kubernetes apiserver has terminated {{ $value | humanizePercentage
          }} of its incoming requests.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests
        summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage
          }} of its incoming requests.
      expr: sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m]))  /
        (  sum(rate(apiserver_request_total{job="kube-apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m]))
        ) > 0.20
      for: 5m
      labels:
        severity: warning
        exported_instance: '{{ $labels.cluster }}/apiserver'
        service: kubernetes-system-apiserver
