From 4547efab09c4242e43cd6cdaac8c58dac07c758e Mon Sep 17 00:00:00 2001
From: klinch0 <68821526+klinch0@users.noreply.github.com>
Date: Mon, 13 Jan 2025 20:39:14 +0300
Subject: [PATCH] add cnpg alerts (#558)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added comprehensive monitoring and alerting rules for PostgreSQL
instances.
	- Introduced alerts for:
		- Long-running transactions
		- Backend waiting times
		- Transaction ID age
		- Replication lag
		- Archiving failures
		- Deadlock conflicts
		- Replication status
	- New resource: `PrometheusRule` named `cnpg-default-alerts`.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
---
 .../templates/prometheusrule.yaml             | 101 ++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100644 packages/system/postgres-operator/templates/prometheusrule.yaml

diff --git a/packages/system/postgres-operator/templates/prometheusrule.yaml b/packages/system/postgres-operator/templates/prometheusrule.yaml
new file mode 100644
index 00000000..6bc7cdb2
--- /dev/null
+++ b/packages/system/postgres-operator/templates/prometheusrule.yaml
@@ -0,0 +1,101 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cnpg-default-alerts
+spec:
+  groups:
+  - name: cnp-default.rules
+    rules:
+    - alert: LongRunningTransaction
+      annotations:
+        description: Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }} is taking more than 5 minutes (300 seconds) for a query.
+        summary: A query is taking longer than 5 minutes.
+      expr: |-
+        cnpg_backends_max_tx_duration_seconds > 300
+      for: 1m
+      labels:
+        severity: warning
+        namespace: {{ $labels.namespace }}
+        job: {{ $labels.job }}
+    - alert: BackendsWaiting
+      annotations:
+        description: Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }} has been waiting for longer than 5 minutes
+        summary: If a backend is waiting for longer than 5 minutes
+      expr: |-
+        cnpg_backends_waiting_total > 300
+      for: 1m
+      labels:
+        severity: warning
+        namespace: {{ $labels.namespace }}
+        job: {{ $labels.job }}
+    - alert: PGDatabaseXidAge
+      annotations:
+        description: Over 300,000,000 transactions from frozen xid on Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }}
+        summary: Number of transactions from the frozen XID to the current one
+      expr: |-
+        cnpg_pg_database_xid_age > 300000000
+      for: 1m
+      labels:
+        severity: warning
+        namespace: {{ $labels.namespace }}
+        job: {{ $labels.job }}
+    - alert: PGReplication
+      annotations:
+        description: Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }} Standby is lagging behind by over 300 seconds (5 minutes)
+        summary: The standby is lagging behind the primary
+      expr: |-
+        cnpg_pg_replication_lag > 300
+      for: 1m
+      labels:
+        severity: warning
+        namespace: {{ $labels.namespace }}
+        job: {{ $labels.job }}
+    - alert: LastFailedArchiveTime
+      annotations:
+        description: Archiving failed for Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }}
+        summary: Checks the last time archiving failed. Will be < 0 when it has not failed.
+      expr: |-
+        sum by (namespace, job) (cnpg_pg_stat_archiver_last_failed_time) - sum by (namespace, job) (cnpg_pg_stat_archiver_last_archived_time) > 1
+      for: 1m
+      labels:
+        severity: warning
+        namespace: {{ $labels.namespace }}
+        job: {{ $labels.job }}
+    - alert: DatabaseDeadlockConflicts
+      annotations:
+        description: There are over 10 deadlock conflicts in Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }}
+        summary: Checks the number of database conflicts
+      expr: |-
+        cnpg_pg_stat_database_deadlocks > 10
+      for: 1m
+      labels:
+        severity: warning
+        namespace: {{ $labels.namespace }}
+        job: {{ $labels.job }}
+    - alert: ReplicaFailingReplication
+      annotations:
+        description: Namespace {{ $labels.namespace }} Job {{ $labels.job }} Pod {{ $labels.pod }} Replica is failing to replicate
+        summary: Checks if the replica is failing to replicate
+      expr: |-
+        sum by (namespace, job) (cnpg_pg_replication_in_recovery) - sum by (namespace, job) (cnpg_pg_replication_is_wal_receiver_up) > 0
+      for: 1m
+      labels:
+        severity: warning
+        namespace: {{ $labels.namespace }}
+        job: {{ $labels.job }}
+    - alert: CNPGClusterOffline
+      annotations:
+        summary: CNPG Cluster has no running instances!
+        description: |-
+          CloudNativePG Cluster Namespace: {{ $labels.namespace }} Job: {{ $labels.job }} Pod: {{ $labels.pod }} has no ready instances.
+          Having an offline cluster means your applications will not be able to access the database, leading to
+          potential service disruption and/or data loss.
+        runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterOffline.md
+      expr: |
+        sum by (namespace, pod) (cnpg_collector_up)) OR on() vector(0) == 0
+      for: 5m
+      labels:
+        severity: critical
+        namespace: {{ $labels.namespace }}
+        job: {{ $labels.job }}
+        pod: {{ $labels.pod }}