apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cnpg-default-alerts
spec:
  groups:
  - name: cnp-default.rules
    rules:
    - alert: LongRunningTransaction
      annotations:
        description: Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }} is taking more than 5 minutes (300 seconds) for a query.
        summary: A query is taking longer than 5 minutes.
      expr: |-
        cnpg_backends_max_tx_duration_seconds > 300
      for: 1m
      labels:
        severity: warning
        namespace: "{{ $labels.namespace }}"
        job: "{{ $labels.job }}"
    - alert: BackendsWaiting
      annotations:
        description: Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }} has been waiting for longer than 5 minutes
        summary: If a backend is waiting for longer than 5 minutes
      expr: |-
        cnpg_backends_waiting_total > 300
      for: 1m
      labels:
        severity: warning
        namespace: "{{ $labels.namespace }}"
        job: "{{ $labels.job }}"
    - alert: PGDatabaseXidAge
      annotations:
        description: Over 300,000,000 transactions from frozen xid on Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }}
        summary: Number of transactions from the frozen XID to the current one
      expr: |-
        cnpg_pg_database_xid_age > 300000000
      for: 1m
      labels:
        severity: warning
        namespace: "{{ $labels.namespace }}"
        job: "{{ $labels.job }}"
    - alert: PGReplication
      annotations:
        description: Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }} Standby is lagging behind by over 300 seconds (5 minutes)
        summary: The standby is lagging behind the primary
      expr: |-
        cnpg_pg_replication_lag > 300
      for: 1m
      labels:
        severity: warning
        namespace: "{{ $labels.namespace }}"
        job: "{{ $labels.job }}"
    - alert: LastFailedArchiveTime
      annotations:
        description: Archiving failed for Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }}
        summary: Checks the last time archiving failed. Will be < 0 when it has not failed.
      expr: |-
        sum by (namespace, job) (cnpg_pg_stat_archiver_last_failed_time) - sum by (namespace, job) (cnpg_pg_stat_archiver_last_archived_time) > 1
      for: 1m
      labels:
        severity: warning
        namespace: "{{ $labels.namespace }}"
        job: "{{ $labels.job }}"
    - alert: DatabaseDeadlockConflicts
      annotations:
        description: There are over 10 deadlock conflicts in Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }}
        summary: Checks the number of database conflicts
      expr: |-
        cnpg_pg_stat_database_deadlocks > 10
      for: 1m
      labels:
        severity: warning
        namespace: "{{ $labels.namespace }}"
        job: "{{ $labels.job }}"
    - alert: ReplicaFailingReplication
      annotations:
        description: Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }} Replica is failing to replicate
        summary: Checks if the replica is failing to replicate
      expr: |-
        sum by (namespace, job) (cnpg_pg_replication_in_recovery) - sum by (namespace, job) (cnpg_pg_replication_is_wal_receiver_up) > 0
      for: 1m
      labels:
        severity: warning
        namespace: "{{ $labels.namespace }}"
        job: "{{ $labels.job }}"
    - alert: CNPGClusterOffline
      annotations:
        summary: CNPG Cluster has no running instances!
        description: |-
          CloudNativePG Cluster Namespace: {{ $labels.namespace }} Job: {{ $labels.job }} Pod: {{ $labels.pod }} has no ready instances.
          Having an offline cluster means your applications will not be able to access the database, leading to
          potential service disruption and/or data loss.
        runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterOffline.md
      expr: |
        sum by (namespace, pod) (cnpg_collector_up)) OR on() vector(0) == 0
      for: 5m
      labels:
        severity: critical
        namespace: "{{ $labels.namespace }}"
        job: "{{ $labels.job }}"
        pod: "{{ $labels.pod }}"
