apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: etcd-rules
spec:
  groups:
  - name: etcd
    rules:
    - alert: etcdInsufficientMembers
      annotations:
        summary: "etcd cluster '{{`{{ $labels.job }}`}}': insufficient members '{{`{{ $value }}`}}'."
      expr: |
        sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)
      for: 3m
      labels:
        severity: critical

    - alert: etcdNoLeader
      annotations:
        summary: "etcd cluster '{{`{{ $labels.job }}`}}': member '{{`{{ $labels.instance }}`}}' has no leader."
      expr: |
        etcd_server_has_leader{job=~".*etcd.*"} == 0
      for: 1m
      labels:
        severity: critical

    - alert: etcdHighNumberOfLeaderChanges
      annotations:
        summary: "etcd cluster '{{`{{ $labels.job }}`}}': instance '{{`{{ $labels.instance }}`}}' has seen '{{`{{ $value }}`}}' leader changes within the last hour."
      expr: |
        rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
      for: 15m
      labels:
        severity: warning

    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
        summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
      expr: |
        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
          /
        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
          > 1
      for: 10m
      labels:
        severity: warning

    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
        summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
      expr: |
        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
          /
        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
          > 5
      for: 5m
      labels:
        severity: critical

    - alert: etcdGRPCRequestsSlow
      annotations:
        summary: "etcd cluster '{{`{{ $labels.job }}`}}': gRPC requests to '{{`{{ $labels.grpc_method }}`}}' are taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'."
      expr: |
        histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
        > 0.15
      for: 10m
      labels:
        severity: critical

    - alert: etcdMemberCommunicationSlow
      annotations:
        summary: "etcd cluster '{{`{{ $labels.job }}`}}': member communication with '{{`{{ $labels.To }}`}}' is taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'."
      expr: |
        histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.15
      for: 10m
      labels:
        severity: warning

    - alert: etcdHighNumberOfFailedProposals
      annotations:
        summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' proposal failures within the last hour on etcd instance '{{`{{ $labels.instance }}`}}'."
      expr: |
        rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
      for: 15m
      labels:
        severity: warning

    - alert: etcdHighNumberOfFailedHTTPRequests
      annotations:
        summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
      expr: |
        sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.01
      for: 10m
      labels:
        severity: warning

    - alert: etcdHighNumberOfFailedHTTPRequests
      annotations:
        summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
      expr: |
        sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.05
      for: 10m
      labels:
        severity: critical

    - alert: etcdHTTPRequestsSlow
      annotations:
        summary: "etcd instance '{{`{{ $labels.instance }}`}}' HTTP requests to '{{`{{ $labels.method }}`}}' are slow."
      expr: |
        histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
        > 0.15
      for: 10m
      labels:
        severity: warning

    - alert: etcdMembersDown
      annotations:
        summary: "etcd cluster '{{`{{ $labels.job }}`}}' members are down."
        description: 'etcd cluster "{{`{{ $labels.job }}`}}": members are down {{`{{ $value }}`}}.'
      expr: |
        max without (endpoint) (
          sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0)
        or
          count without (To) (
            sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
          )
        )
        > 0
      for: 10m
      labels:
        severity: critical
