Upd seaweedfs

This PR introduces an extra option to cozystack configmap. It allows to
enable components that are optional to specific bundle name, example
usage:

```yaml
bundle-enable: telepresence,external-dns,external-secrets-operator
```

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added optional attributes to several release entries, allowing for
more flexible deployment configurations.
- Enhanced Helm release generation logic to consider both enabled and
disabled states for components.
- Improved namespace management by incorporating checks for optional
components based on enablement and disablement.

- **Bug Fixes**
- Updated dependencies for `external-dns` to include `cilium` and
`kubeovn`.

- **Documentation**
- Clarified the configurability of deployment components with the
introduction of optional attributes.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

.gitignore

@@ -1,3 +1,78 @@
 _out
 .git
-.idea
+.idea
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+.DS_Store
+**/.DS_Store

Makefile

@@ -2,11 +2,15 @@
 
 build:
 	make -C packages/apps/http-cache image
+	make -C packages/apps/postgres image
+	make -C packages/apps/mysql image
+	make -C packages/apps/clickhouse image
 	make -C packages/apps/kubernetes image
 	make -C packages/system/cilium image
 	make -C packages/system/kubeovn image
 	make -C packages/system/dashboard image
 	make -C packages/system/kamaji image
+	make -C packages/core/testing image
 	make -C packages/core/installer image
 	make manifests
 
@@ -26,3 +30,8 @@ repos:
 
 assets:
 	make -C packages/core/installer/ assets
+
+test:
+	make -C packages/core/testing apply
+	make -C packages/core/testing test
+	make -C packages/core/testing delete

README.md

@@ -58,6 +58,8 @@ Commits are used to generate the changelog, and their author will be referenced
 
 In case of **Feature Requests** please use the [Discussion's Feature Request section](https://github.com/aenix-io/cozystack/discussions/categories/feature-requests).
 
+You can join our weekly community meetings (just add this events to your [Google Calendar](https://calendar.google.com/calendar?cid=ZTQzZDIxZTVjOWI0NWE5NWYyOGM1ZDY0OWMyY2IxZTFmNDMzZTJlNjUzYjU2ZGJiZGE3NGNhMzA2ZjBkMGY2OEBncm91cC5jYWxlbmRhci5nb29nbGUuY29t) or [iCal](https://calendar.google.com/calendar/ical/e43d21e5c9b45a95f28c5d649c2cb1e1f433e2e653b56dbbda74ca306f0d0f68%40group.calendar.google.com/public/basic.ics)) or [Telegram group](https://t.me/cozystack).
+
 ## License
 
 Cozystack is licensed under Apache 2.0.  

dashboards/control-plane/deprecated-resources.json

@@ -590,6 +590,25 @@
         "skipUrlSync": false,
         "sort": 0,
         "type": "query"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "hide": 2,
+        "includeAll": false,
+        "label": "Prometheus",
+        "multi": false,
+        "name": "ds_prometheus",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
       }
     ]
   },

dashboards/dotdc/k8s-system-coredns.json

@@ -120,9 +120,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "value_and_name"
+        "showPercentChange": false,
+        "textMode": "value_and_name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -130,7 +132,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "up{job=\"coredns\", instance=~\"$instance\"}",
+          "expr": "up{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}",
           "interval": "",
           "legendFormat": "{{ instance }}",
           "refId": "A"
@@ -150,6 +152,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -163,6 +166,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -225,7 +229,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "rate(process_cpu_seconds_total{job=\"coredns\", instance=~\"$instance\"}[$__rate_interval])",
+          "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}[$__rate_interval])",
           "interval": "$resolution",
           "legendFormat": "{{ instance }}",
           "refId": "A"
@@ -245,6 +249,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -258,6 +263,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -319,7 +325,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "process_resident_memory_bytes{job=\"coredns\", instance=~\"$instance\"}",
+          "expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}",
           "interval": "",
           "legendFormat": "{{ instance }}",
           "refId": "A"
@@ -339,6 +345,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -352,6 +359,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -413,7 +421,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(coredns_dns_requests_total{instance=~\"$instance\",proto=\"$protocol\"}[$__rate_interval]))",
+          "expr": "sum(rate(coredns_dns_requests_total{instance=~\"$instance\",proto=\"$protocol\", cluster=~\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "total $protocol requests",
           "refId": "A"
@@ -433,6 +441,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -446,6 +455,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -507,7 +517,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(coredns_dns_request_size_bytes_sum{instance=~\"$instance\",proto=\"$protocol\"}[$__rate_interval])) by (proto) / sum(rate(coredns_dns_request_size_bytes_count{instance=~\"$instance\",proto=\"$protocol\"}[$__rate_interval])) by (proto)",
+          "expr": "sum(rate(coredns_dns_request_size_bytes_sum{instance=~\"$instance\",proto=\"$protocol\", cluster=~\"$cluster\"}[$__rate_interval])) by (proto) / sum(rate(coredns_dns_request_size_bytes_count{instance=~\"$instance\",proto=\"$protocol\", cluster=~\"$cluster\"}[$__rate_interval])) by (proto)",
           "interval": "$resolution",
           "legendFormat": "average $protocol packet size",
           "refId": "A"
@@ -527,6 +537,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -540,6 +551,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -601,7 +613,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(coredns_dns_requests_total{instance=~\"$instance\"}[$__rate_interval])) by (type)",
+          "expr": "sum(rate(coredns_dns_requests_total{instance=~\"$instance\", cluster=~\"$cluster\"}[$__rate_interval])) by (type)",
           "interval": "$resolution",
           "legendFormat": "{{ type }}",
           "refId": "A"
@@ -621,6 +633,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -634,6 +647,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -695,7 +709,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(coredns_dns_responses_total{instance=~\"$instance\"}[$__rate_interval])) by (rcode)",
+          "expr": "sum(rate(coredns_dns_responses_total{instance=~\"$instance\", cluster=~\"$cluster\"}[$__rate_interval])) by (rcode)",
           "interval": "$resolution",
           "legendFormat": "{{ rcode }}",
           "refId": "A"
@@ -715,6 +729,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -728,6 +743,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -789,7 +805,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(coredns_forward_requests_total[$__rate_interval]))",
+          "expr": "sum(rate(coredns_forward_requests_total{cluster=~\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "total forward requests",
           "refId": "A"
@@ -809,6 +825,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -822,6 +839,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -883,7 +901,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(coredns_forward_responses_total{rcode=~\"SERVFAIL|REFUSED\"}[$__rate_interval])) by (rcode)",
+          "expr": "sum(rate(coredns_forward_responses_total{rcode=~\"SERVFAIL|REFUSED\", cluster=~\"$cluster\"}[$__rate_interval])) by (rcode)",
           "interval": "$resolution",
           "legendFormat": "{{ rcode }}",
           "refId": "A"
@@ -903,6 +921,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -916,6 +935,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -977,7 +997,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(coredns_cache_hits_total{instance=~\"$instance\"}[$__rate_interval])) by (type)",
+          "expr": "sum(rate(coredns_cache_hits_total{instance=~\"$instance\", cluster=~\"$cluster\"}[$__rate_interval])) by (type)",
           "interval": "$resolution",
           "legendFormat": "{{ type }}",
           "refId": "A"
@@ -988,7 +1008,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(coredns_cache_misses_total{instance=~\"$instance\"}[$__rate_interval])) by (type)",
+          "expr": "sum(rate(coredns_cache_misses_total{instance=~\"$instance\", cluster=~\"$cluster\"}[$__rate_interval])) by (type)",
           "interval": "$resolution",
           "legendFormat": "misses",
           "refId": "B"
@@ -1008,6 +1028,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -1021,6 +1042,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 2,
             "pointSize": 5,
@@ -1082,7 +1104,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(coredns_cache_entries) by (type)",
+          "expr": "sum(coredns_cache_entries{cluster=~\"$cluster\"}) by (type)",
           "interval": "",
           "legendFormat": "{{ type }}",
           "refId": "A"
@@ -1143,7 +1165,8 @@
           "layout": "auto"
         },
         "tooltip": {
-          "show": true,
+          "mode": "single",
+          "showColorScale": false,
           "yHistogram": false
         },
         "yAxis": {
@@ -1152,7 +1175,7 @@
           "unit": "s"
         }
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -1160,7 +1183,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(increase(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])) by (le)",
+          "expr": "sum(increase(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\", cluster=~\"$cluster\"}[$__rate_interval])) by (le)",
           "format": "heatmap",
           "legendFormat": "{{le}}",
           "range": true,
@@ -1196,85 +1219,6 @@
         "x": 12,
         "y": 43
       },
-      "id": 30,
-      "options": {
-        "calculate": false,
-        "cellGap": 1,
-        "color": {
-          "exponent": 0.5,
-          "fill": "dark-orange",
-          "mode": "scheme",
-          "reverse": false,
-          "scale": "exponential",
-          "scheme": "RdYlBu",
-          "steps": 64
-        },
-        "exemplars": {
-          "color": "rgba(255,0,255,0.7)"
-        },
-        "filterValues": {
-          "le": 1e-9
-        },
-        "legend": {
-          "show": true
-        },
-        "rowsFrame": {
-          "layout": "auto"
-        },
-        "tooltip": {
-          "show": true,
-          "yHistogram": false
-        },
-        "yAxis": {
-          "axisPlacement": "left",
-          "reverse": false,
-          "unit": "s"
-        }
-      },
-      "pluginVersion": "10.0.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "editorMode": "code",
-          "expr": "sum(increase(coredns_forward_request_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])) by (le)",
-          "format": "heatmap",
-          "legendFormat": "{{le}}",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "CoreDNS - Forward request duration",
-      "type": "heatmap"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${datasource}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "scaleDistribution": {
-              "type": "linear"
-            }
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 53
-      },
       "id": 28,
       "options": {
         "calculate": false,
@@ -1301,7 +1245,8 @@
           "layout": "auto"
         },
         "tooltip": {
-          "show": true,
+          "mode": "single",
+          "showColorScale": false,
           "yHistogram": false
         },
         "yAxis": {
@@ -1310,7 +1255,7 @@
           "unit": "decbytes"
         }
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -1318,7 +1263,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(increase(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\", le!=\"0\"}[$__rate_interval])) by (le)",
+          "expr": "sum(increase(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\", le!=\"0\", cluster=~\"$cluster\"}[$__rate_interval])) by (le)",
           "format": "heatmap",
           "legendFormat": "{{le}}",
           "range": true,
@@ -1351,7 +1296,7 @@
       "gridPos": {
         "h": 10,
         "w": 12,
-        "x": 12,
+        "x": 0,
         "y": 53
       },
       "id": 29,
@@ -1380,7 +1325,8 @@
           "layout": "auto"
         },
         "tooltip": {
-          "show": true,
+          "mode": "single",
+          "showColorScale": false,
           "yHistogram": false
         },
         "yAxis": {
@@ -1389,7 +1335,7 @@
           "unit": "decbytes"
         }
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -1397,7 +1343,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(increase(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\", le!=\"0\"}[$__rate_interval])) by (le)",
+          "expr": "sum(increase(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\", le!=\"0\", cluster=~\"$cluster\"}[$__rate_interval])) by (le)",
           "format": "heatmap",
           "legendFormat": "{{le}}",
           "range": true,
@@ -1409,8 +1355,7 @@
     }
   ],
   "refresh": "30s",
-  "schemaVersion": 38,
-  "style": "dark",
+  "schemaVersion": 39,
   "tags": [
     "Kubernetes",
     "Prometheus"
@@ -1435,6 +1380,34 @@
         "skipUrlSync": false,
         "type": "datasource"
       },
+      {
+        "current": {
+          "isNone": true,
+          "selected": false,
+          "text": "None",
+          "value": ""
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(kube_node_info,cluster)",
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(kube_node_info,cluster)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
       {
         "current": {
           "selected": false,
@@ -1445,7 +1418,7 @@
           "type": "prometheus",
           "uid": "${datasource}"
         },
-        "definition": "label_values(up{job=\"coredns\"}, instance)",
+        "definition": "label_values(up{job=\"$job\", cluster=\"$cluster\"},instance)",
         "hide": 0,
         "includeAll": true,
         "label": "",
@@ -1453,8 +1426,9 @@
         "name": "instance",
         "options": [],
         "query": {
-          "query": "label_values(up{job=\"coredns\"}, instance)",
-          "refId": "StandardVariableQuery"
+          "qryType": 1,
+          "query": "label_values(up{job=\"$job\", cluster=\"$cluster\"},instance)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
         "regex": "",
@@ -1476,7 +1450,7 @@
           "type": "prometheus",
           "uid": "${datasource}"
         },
-        "definition": "label_values(coredns_dns_requests_total, proto)",
+        "definition": "label_values(coredns_dns_requests_total{cluster=\"$cluster\"}, proto)",
         "hide": 0,
         "includeAll": false,
         "label": "",
@@ -1484,7 +1458,7 @@
         "name": "protocol",
         "options": [],
         "query": {
-          "query": "label_values(coredns_dns_requests_total, proto)",
+          "query": "label_values(coredns_dns_requests_total{cluster=\"$cluster\"}, proto)",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -1498,7 +1472,7 @@
       },
       {
         "current": {
-          "selected": true,
+          "selected": false,
           "text": "30s",
           "value": "30s"
         },
@@ -1542,6 +1516,37 @@
         "queryValue": "",
         "skipUrlSync": false,
         "type": "custom"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "coredns"
+          ],
+          "value": [
+            "coredns"
+          ]
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(coredns_build_info{cluster=\"$cluster\"},job)",
+        "hide": 0,
+        "includeAll": false,
+        "multi": true,
+        "name": "job",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(coredns_build_info{cluster=\"$cluster\"},job)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
       }
     ]
   },
@@ -1553,6 +1558,6 @@
   "timezone": "",
   "title": "Kubernetes / System / CoreDNS",
   "uid": "k8s_system_coredns",
-  "version": 13,
+  "version": 18,
   "weekStart": ""
 }

dashboards/dotdc/k8s-views-pods.json

@@ -108,6 +108,7 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
+      "description": "Panel only works when a single pod is selected.",
       "fieldConfig": {
         "defaults": {
           "mappings": [],
@@ -136,6 +137,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -143,17 +145,20 @@
           "fields": "",
           "values": false
         },
-        "textMode": "name"
+        "showPercentChange": false,
+        "textMode": "name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "${datasource}"
           },
+          "editorMode": "code",
           "exemplar": false,
-          "expr": "kube_pod_info{namespace=\"$namespace\", pod=\"$pod\"}",
+          "expr": "kube_pod_info{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}",
           "instant": true,
           "interval": "",
           "legendFormat": "{{ created_by_kind }}: {{ created_by_name }}",
@@ -168,12 +173,13 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
+      "description": "Panel only works when a single pod is selected.",
       "fieldConfig": {
         "defaults": {
           "links": [
             {
               "title": "",
-              "url": "/d/k8s_views_nodes/kubernetes-views-nodes?var-datasource=${datasource}&var-node=${__data.fields.node}"
+              "url": "/d/k8s_views_nodes/kubernetes-views-nodes?var-datasource=${datasource}&var-node=${__field.labels.node}"
             }
           ],
           "mappings": [],
@@ -202,6 +208,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -209,17 +216,20 @@
           "fields": "",
           "values": false
         },
-        "textMode": "name"
+        "showPercentChange": false,
+        "textMode": "name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "${datasource}"
           },
+          "editorMode": "code",
           "exemplar": false,
-          "expr": "kube_pod_info{namespace=\"$namespace\", pod=\"$pod\"}",
+          "expr": "kube_pod_info{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}",
           "instant": true,
           "interval": "",
           "legendFormat": "{{ node }}",
@@ -234,6 +244,7 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
+      "description": "Panel only works when a single pod is selected.",
       "fieldConfig": {
         "defaults": {
           "mappings": [],
@@ -262,6 +273,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -269,17 +281,20 @@
           "fields": "",
           "values": false
         },
-        "textMode": "name"
+        "showPercentChange": false,
+        "textMode": "name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "${datasource}"
           },
+          "editorMode": "code",
           "exemplar": false,
-          "expr": "kube_pod_info{namespace=\"$namespace\", pod=\"$pod\"}",
+          "expr": "kube_pod_info{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}",
           "instant": true,
           "interval": "",
           "legendFormat": "{{ pod_ip }}",
@@ -294,6 +309,7 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
+      "description": "Panel only works when a single pod is selected.",
       "fieldConfig": {
         "defaults": {
           "mappings": [],
@@ -322,6 +338,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -329,9 +346,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "name"
+        "showPercentChange": false,
+        "textMode": "name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
@@ -340,7 +359,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "kube_pod_info{namespace=\"$namespace\", pod=\"$pod\", priority_class!=\"\"}",
+          "expr": "kube_pod_info{namespace=\"$namespace\", pod=\"$pod\", priority_class!=\"\", cluster=\"$cluster\"}",
           "format": "time_series",
           "instant": true,
           "interval": "",
@@ -357,6 +376,7 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
+      "description": "Panel only works when a single pod is selected.",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -419,14 +439,17 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [],
           "fields": "",
           "values": false
         },
-        "textMode": "name"
+        "showPercentChange": false,
+        "textMode": "name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
@@ -435,7 +458,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "kube_pod_status_qos_class{namespace=\"$namespace\", pod=\"$pod\"} > 0",
+          "expr": "kube_pod_status_qos_class{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"} > 0",
           "instant": true,
           "interval": "",
           "legendFormat": "{{ qos_class }}",
@@ -450,6 +473,7 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
+      "description": "Panel only works when a single pod is selected.",
       "fieldConfig": {
         "defaults": {
           "mappings": [],
@@ -482,14 +506,17 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [],
           "fields": "",
           "values": false
         },
-        "textMode": "name"
+        "showPercentChange": false,
+        "textMode": "name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
@@ -498,7 +525,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "kube_pod_container_status_last_terminated_reason{namespace=\"$namespace\", pod=\"$pod\"}",
+          "expr": "kube_pod_container_status_last_terminated_reason{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}",
           "instant": true,
           "interval": "",
           "legendFormat": "{{ reason }}",
@@ -513,6 +540,7 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
+      "description": "Panel only works when a single pod is selected.",
       "fieldConfig": {
         "defaults": {
           "mappings": [],
@@ -549,14 +577,17 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [],
           "fields": "",
           "values": true
         },
-        "textMode": "value"
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
@@ -565,7 +596,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "kube_pod_container_status_last_terminated_exitcode{namespace=\"$namespace\", pod=\"$pod\"}",
+          "expr": "kube_pod_container_status_last_terminated_exitcode{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}",
           "instant": true,
           "interval": "",
           "legendFormat": "__auto",
@@ -646,6 +677,8 @@
       },
       "id": 39,
       "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
         "orientation": "auto",
         "reduceOptions": {
           "calcs": [
@@ -655,9 +688,10 @@
           "values": false
         },
         "showThresholdLabels": false,
-        "showThresholdMarkers": true
+        "showThresholdMarkers": true,
+        "sizing": "auto"
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
@@ -666,7 +700,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}[$__rate_interval])) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"})",
+          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", cluster=\"$cluster\"}[$__rate_interval])) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"})",
           "instant": true,
           "interval": "$resolution",
           "legendFormat": "Requests",
@@ -716,6 +750,8 @@
       },
       "id": 48,
       "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
         "orientation": "auto",
         "reduceOptions": {
           "calcs": [
@@ -725,9 +761,10 @@
           "values": false
         },
         "showThresholdLabels": false,
-        "showThresholdMarkers": true
+        "showThresholdMarkers": true,
+        "sizing": "auto"
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
@@ -736,7 +773,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}[$__rate_interval])) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"})",
+          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", cluster=\"$cluster\"}[$__rate_interval])) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"})",
           "instant": true,
           "interval": "$resolution",
           "legendFormat": "Limits",
@@ -790,6 +827,8 @@
       },
       "id": 40,
       "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
         "orientation": "auto",
         "reduceOptions": {
           "calcs": [
@@ -799,17 +838,19 @@
           "values": false
         },
         "showThresholdLabels": false,
-        "showThresholdMarkers": true
+        "showThresholdMarkers": true,
+        "sizing": "auto"
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "${datasource}"
           },
+          "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"})",
+          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", cluster=\"$cluster\"}) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"})",
           "instant": true,
           "interval": "$resolution",
           "legendFormat": "Requests",
@@ -859,6 +900,8 @@
       },
       "id": 49,
       "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
         "orientation": "auto",
         "reduceOptions": {
           "calcs": [
@@ -868,17 +911,19 @@
           "values": false
         },
         "showThresholdLabels": false,
-        "showThresholdMarkers": true
+        "showThresholdMarkers": true,
+        "sizing": "auto"
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "${datasource}"
           },
+          "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}) ",
+          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", cluster=\"$cluster\"}) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) ",
           "instant": true,
           "interval": "$resolution",
           "legendFormat": "Limits",
@@ -988,7 +1033,7 @@
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "10.1.0",
+      "pluginVersion": "11.2.0",
       "targets": [
         {
           "datasource": {
@@ -997,7 +1042,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)",
+          "expr": "sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (container)",
           "format": "table",
           "instant": true,
           "interval": "",
@@ -1012,7 +1057,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)",
+          "expr": "sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (container)",
           "format": "table",
           "instant": true,
           "interval": "",
@@ -1027,7 +1072,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}) by (container)",
+          "expr": "sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (container)",
           "format": "table",
           "instant": true,
           "interval": "",
@@ -1041,7 +1086,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}) by (container)",
+          "expr": "sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (container)",
           "format": "table",
           "instant": true,
           "interval": "",
@@ -1055,7 +1100,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\"}[$__rate_interval])) by (container)",
+          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -1070,7 +1115,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\"}) by (container)",
+          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}) by (container)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -1181,11 +1226,13 @@
             "mode": "thresholds"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "Percent",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -1271,7 +1318,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)",
+          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (container)",
           "interval": "$resolution",
           "legendFormat": "{{ container }}  REQUESTS",
           "range": true,
@@ -1283,7 +1330,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)",
+          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$pod\", resource=\"cpu\", job=~\"$job\", cluster=\"$cluster\"}) by (container)",
           "hide": false,
           "legendFormat": "{{ container }}  LIMITS",
           "range": true,
@@ -1305,11 +1352,13 @@
             "mode": "thresholds"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "Percent",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -1398,7 +1447,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}) by (container)",
+          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", cluster=\"$cluster\"}) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (container)",
           "interval": "",
           "legendFormat": "{{ container }} REQUESTS",
           "range": true,
@@ -1410,7 +1459,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}) by (container)",
+          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", cluster=\"$cluster\"}) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"$pod\", resource=\"memory\", job=~\"$job\", cluster=\"$cluster\"}) by (container)",
           "hide": false,
           "legendFormat": "{{ container }} LIMITS",
           "range": true,
@@ -1431,11 +1480,13 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "CPU Cores",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -1532,7 +1583,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\"}[$__rate_interval])) by (container)",
+          "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)",
           "interval": "$resolution",
           "legendFormat": "{{ container }}",
           "range": true,
@@ -1553,11 +1604,13 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "Bytes",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -1637,7 +1690,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\"}) by (container)",
+          "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}) by (container)",
           "interval": "",
           "legendFormat": "{{ container }}",
           "range": true,
@@ -1659,11 +1712,13 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "SECONDS",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -1745,7 +1800,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"\"}[$__rate_interval])) by (container)",
+          "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\", pod=~\"$pod\", image!=\"\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)",
           "interval": "$resolution",
           "legendFormat": "{{ container }}",
           "range": true,
@@ -1780,11 +1835,13 @@
             "mode": "thresholds"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "Percent",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -1873,7 +1930,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(increase(container_oom_events_total{namespace=\"${namespace}\", pod=\"${pod}\", container!=\"\"}[$__rate_interval])) by (container)",
+          "expr": "sum(increase(container_oom_events_total{namespace=\"${namespace}\", pod=\"${pod}\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)",
           "interval": "",
           "legendFormat": "{{ container }}",
           "range": true,
@@ -1895,11 +1952,13 @@
             "mode": "thresholds"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "Percent",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -1988,7 +2047,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"${namespace}\", pod=\"${pod}\", container!=\"\"}[$__rate_interval])) by (container)",
+          "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"${namespace}\", pod=\"${pod}\", container!=\"\", job=~\"$job\", cluster=\"$cluster\"}[$__rate_interval])) by (container)",
           "interval": "",
           "legendFormat": "{{ container }}",
           "range": true,
@@ -2035,11 +2094,13 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -2079,7 +2140,7 @@
               }
             ]
           },
-          "unit": "bytes"
+          "unit": "binBps"
         },
         "overrides": []
       },
@@ -2110,7 +2171,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+          "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\", pod=~\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "Received",
           "refId": "A"
@@ -2121,7 +2182,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "- sum(rate(container_network_transmit_bytes_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+          "expr": "- sum(rate(container_network_transmit_bytes_total{namespace=\"$namespace\", pod=~\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "Transmitted",
           "refId": "B"
@@ -2141,11 +2202,13 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -2216,7 +2279,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(container_network_receive_packets_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+          "expr": "sum(rate(container_network_receive_packets_total{namespace=\"$namespace\", pod=~\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "Received",
           "refId": "A"
@@ -2227,7 +2290,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "- sum(rate(container_network_transmit_packets_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+          "expr": "- sum(rate(container_network_transmit_packets_total{namespace=\"$namespace\", pod=~\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "Transmitted",
           "refId": "B"
@@ -2247,11 +2310,13 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -2282,7 +2347,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2321,7 +2387,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(container_network_receive_packets_dropped_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+          "expr": "sum(rate(container_network_receive_packets_dropped_total{namespace=\"$namespace\", pod=~\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "Received",
           "refId": "A"
@@ -2332,7 +2398,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "- sum(rate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+          "expr": "- sum(rate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\", pod=~\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "Transmitted",
           "refId": "B"
@@ -2352,11 +2418,13 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 25,
             "gradientMode": "opacity",
@@ -2387,7 +2455,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2426,7 +2495,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "sum(rate(container_network_receive_errors_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+          "expr": "sum(rate(container_network_receive_errors_total{namespace=\"$namespace\", pod=~\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "Received",
           "refId": "A"
@@ -2437,7 +2506,7 @@
             "uid": "${datasource}"
           },
           "exemplar": true,
-          "expr": "- sum(rate(container_network_transmit_errors_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+          "expr": "- sum(rate(container_network_transmit_errors_total{namespace=\"$namespace\", pod=~\"$pod\", cluster=\"$cluster\"}[$__rate_interval]))",
           "interval": "$resolution",
           "legendFormat": "Transmitted",
           "refId": "B"
@@ -2448,8 +2517,7 @@
     }
   ],
   "refresh": "30s",
-  "schemaVersion": 38,
-  "style": "dark",
+  "schemaVersion": 39,
   "tags": [
     "Kubernetes",
     "Prometheus"
@@ -2459,8 +2527,8 @@
       {
         "current": {
           "selected": false,
-          "text": "Prometheus",
-          "value": "Prometheus"
+          "text": "",
+          "value": ""
         },
         "hide": 0,
         "includeAll": false,
@@ -2474,6 +2542,34 @@
         "skipUrlSync": false,
         "type": "datasource"
       },
+      {
+        "current": {
+          "isNone": true,
+          "selected": false,
+          "text": "None",
+          "value": ""
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(kube_node_info,cluster)",
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(kube_node_info,cluster)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
       {
         "current": {
           "selected": false,
@@ -2484,14 +2580,14 @@
           "type": "prometheus",
           "uid": "${datasource}"
         },
-        "definition": "label_values(kube_pod_info, namespace)",
+        "definition": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)",
         "hide": 0,
         "includeAll": false,
         "multi": false,
         "name": "namespace",
         "options": [],
         "query": {
-          "query": "label_values(kube_pod_info, namespace)",
+          "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)",
           "refId": "Prometheus-namespace-Variable-Query"
         },
         "refresh": 1,
@@ -2513,14 +2609,14 @@
           "type": "prometheus",
           "uid": "${datasource}"
         },
-        "definition": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)",
+        "definition": "label_values(kube_pod_info{namespace=\"$namespace\", cluster=\"$cluster\"}, pod)",
         "hide": 0,
-        "includeAll": false,
-        "multi": false,
+        "includeAll": true,
+        "multi": true,
         "name": "pod",
         "options": [],
         "query": {
-          "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)",
+          "query": "label_values(kube_pod_info{namespace=\"$namespace\", cluster=\"$cluster\"}, pod)",
           "refId": "Prometheus-pod-Variable-Query"
         },
         "refresh": 2,
@@ -2534,7 +2630,7 @@
       },
       {
         "current": {
-          "selected": true,
+          "selected": false,
           "text": "30s",
           "value": "30s"
         },
@@ -2578,6 +2674,33 @@
         "queryValue": "",
         "skipUrlSync": false,
         "type": "custom"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "kube-state-metrics",
+          "value": "kube-state-metrics"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(kube_pod_info{namespace=\"$namespace\", cluster=\"$cluster\"},job)",
+        "hide": 0,
+        "includeAll": false,
+        "multi": true,
+        "name": "job",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(kube_pod_info{namespace=\"$namespace\", cluster=\"$cluster\"},job)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
       }
     ]
   },
@@ -2589,6 +2712,6 @@
   "timezone": "",
   "title": "Kubernetes / Views / Pods",
   "uid": "k8s_views_pods",
-  "version": 22,
+  "version": 30,
   "weekStart": ""
 }

dashboards/ingress/namespace-detail.json

@@ -1339,11 +1339,7 @@
               },
               {
                 "id": "unit",
-                "value": "short"
-              },
-              {
-                "id": "decimals",
-                "value": 2
+                "value": "none"
               },
               {
                 "id": "custom.align",

dashboards/victoria-metrics/backupmanager.json

@@ -12,7 +12,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "9.0.4"
+      "version": "10.4.0"
     },
     {
       "type": "datasource",
@@ -124,9 +124,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.0.4",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -198,9 +200,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.0.4",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -260,9 +264,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.0.4",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -323,9 +329,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.0.4",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -399,9 +407,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.0.4",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -471,9 +481,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.0.4",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -546,9 +558,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.0.4",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -577,7 +591,9 @@
           },
           "custom": {
             "align": "auto",
-            "displayMode": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
             "inspect": false
           },
           "mappings": [],
@@ -630,7 +646,9 @@
       },
       "id": 22,
       "options": {
+        "cellHeight": "sm",
         "footer": {
+          "countRows": false,
           "fields": "",
           "reducer": [
             "sum"
@@ -645,7 +663,7 @@
           }
         ]
       },
-      "pluginVersion": "9.0.4",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -700,7 +718,9 @@
           },
           "custom": {
             "align": "auto",
-            "displayMode": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
             "inspect": false
           },
           "mappings": [],
@@ -753,7 +773,9 @@
       },
       "id": 21,
       "options": {
+        "cellHeight": "sm",
         "footer": {
+          "countRows": false,
           "fields": "",
           "reducer": [
             "sum"
@@ -768,7 +790,7 @@
           }
         ]
       },
-      "pluginVersion": "9.0.4",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -885,7 +907,7 @@
                 "min",
                 "mean"
               ],
-              "displayMode": "table",
+              "displayMode": "list",
               "placement": "bottom",
               "showLegend": false
             },
@@ -1106,7 +1128,9 @@
               },
               "custom": {
                 "align": "auto",
-                "displayMode": "auto",
+                "cellOptions": {
+                  "type": "auto"
+                },
                 "inspect": false
               },
               "mappings": [],
@@ -1251,7 +1275,8 @@
             "legend": {
               "calcs": [],
               "displayMode": "list",
-              "placement": "bottom"
+              "placement": "bottom",
+              "showLegend": true
             },
             "orientation": "auto",
             "showValue": "auto",
@@ -1343,7 +1368,7 @@
                 "max",
                 "mean"
               ],
-              "displayMode": "table",
+              "displayMode": "list",
               "placement": "right",
               "showLegend": false
             },
@@ -1436,7 +1461,8 @@
             "legend": {
               "calcs": [],
               "displayMode": "list",
-              "placement": "bottom"
+              "placement": "bottom",
+              "showLegend": true
             },
             "orientation": "auto",
             "showValue": "auto",
@@ -1657,8 +1683,7 @@
     }
   ],
   "refresh": "1m",
-  "schemaVersion": 36,
-  "style": "dark",
+  "schemaVersion": 39,
   "tags": [],
   "templating": {
     "list": [

dashboards/victoria-metrics/vmalert.json

@@ -6,7 +6,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "9.2.7"
+      "version": "10.4.2"
     },
     {
       "type": "datasource",
@@ -59,7 +59,7 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(short_version))",
+        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
         "hide": true,
         "iconColor": "dark-blue",
         "name": "version",
@@ -72,15 +72,14 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"})) by(job, instance)",
-        "hide": true,
+        "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job, instance)",
         "iconColor": "dark-yellow",
         "name": "restarts",
         "textFormat": "{{job}}:{{instance}} restarted"
       }
     ]
   },
-  "description": "Overview for VictoriaMetrics vmalert v1.96.0 or higher",
+  "description": "Overview for VictoriaMetrics vmalert v1.102.0 or higher",
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
@@ -96,7 +95,7 @@
       "title": "vmalert docs",
       "tooltip": "",
       "type": "link",
-      "url": "https://docs.victoriametrics.com/vmalert.html"
+      "url": "https://docs.victoriametrics.com/vmalert/"
     },
     {
       "asDropdown": false,
@@ -201,10 +200,12 @@
           "fields": "",
           "values": false
         },
+        "showPercentChange": false,
         "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.2.7",
+      "pluginVersion": "10.4.2",
       "targets": [
         {
           "datasource": {
@@ -261,10 +262,12 @@
           "fields": "",
           "values": false
         },
+        "showPercentChange": false,
         "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.2.7",
+      "pluginVersion": "10.4.2",
       "targets": [
         {
           "datasource": {
@@ -321,10 +324,12 @@
           "fields": "",
           "values": false
         },
+        "showPercentChange": false,
         "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.2.7",
+      "pluginVersion": "10.4.2",
       "targets": [
         {
           "datasource": {
@@ -385,10 +390,12 @@
           "fields": "",
           "values": false
         },
+        "showPercentChange": false,
         "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.2.7",
+      "pluginVersion": "10.4.2",
       "targets": [
         {
           "datasource": {
@@ -449,10 +456,12 @@
           "fields": "",
           "values": false
         },
+        "showPercentChange": false,
         "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.2.7",
+      "pluginVersion": "10.4.2",
       "targets": [
         {
           "datasource": {
@@ -483,7 +492,9 @@
           },
           "custom": {
             "align": "auto",
-            "displayMode": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
             "inspect": false,
             "minWidth": 50
           },
@@ -537,7 +548,9 @@
       },
       "id": 45,
       "options": {
+        "cellHeight": "sm",
         "footer": {
+          "countRows": false,
           "fields": "",
           "reducer": [
             "sum"
@@ -546,7 +559,7 @@
         },
         "showHeader": true
       },
-      "pluginVersion": "9.2.7",
+      "pluginVersion": "10.4.2",
       "targets": [
         {
           "datasource": {
@@ -575,6 +588,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -588,6 +602,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "stepAfter",
             "lineWidth": 1,
             "pointSize": 5,
@@ -706,6 +721,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -719,6 +735,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -809,6 +826,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -822,6 +840,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -912,6 +931,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -925,6 +945,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -1013,6 +1034,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -1026,6 +1048,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -1114,7 +1137,9 @@
           },
           "custom": {
             "align": "auto",
-            "displayMode": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
             "inspect": false
           },
           "mappings": [],
@@ -1122,8 +1147,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1234,7 +1258,7 @@
         "type": "prometheus",
         "uid": "$ds"
       },
-      "description": "Missed evaluation means that group evaluation time takes longer than the configured evaluation interval. \nThis may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency for such groups. See https://docs.victoriametrics.com/vmalert.html#groups\n\nIf rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/Troubleshooting.html#slow-queries.\"",
+      "description": "Missed evaluation means that group evaluation time takes longer than the configured evaluation interval. \nThis may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency for such groups. See https://docs.victoriametrics.com/vmalert/#groups\n\nIf rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/troubleshooting/#slow-queries.\"",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1275,8 +1299,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1356,6 +1379,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -1369,6 +1393,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -1400,7 +1425,8 @@
                   }
                 ]
               },
-              "unit": "percentunit"
+              "unit": "percentunit",
+              "unitScale": true
             },
             "overrides": []
           },
@@ -1408,14 +1434,14 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 33
+            "y": 3
           },
           "id": 37,
           "links": [
             {
               "targetBlank": true,
               "title": "Profiling",
-              "url": "https://docs.victoriametrics.com/vmagent.html#profiling"
+              "url": "https://docs.victoriametrics.com/vmagent/#profiling"
             }
           ],
           "options": {
@@ -1467,6 +1493,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -1480,6 +1507,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -1511,7 +1539,8 @@
                   }
                 ]
               },
-              "unit": "bytes"
+              "unit": "bytes",
+              "unitScale": true
             },
             "overrides": []
           },
@@ -1519,14 +1548,14 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 33
+            "y": 3
           },
           "id": 57,
           "links": [
             {
               "targetBlank": true,
               "title": "Profiling",
-              "url": "https://docs.victoriametrics.com/vmagent.html#profiling"
+              "url": "https://docs.victoriametrics.com/vmagent/#profiling"
             }
           ],
           "options": {
@@ -1578,6 +1607,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -1591,6 +1621,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -1622,7 +1653,8 @@
                   }
                 ]
               },
-              "unit": "percentunit"
+              "unit": "percentunit",
+              "unitScale": true
             },
             "overrides": []
           },
@@ -1630,14 +1662,14 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 41
+            "y": 11
           },
           "id": 35,
           "links": [
             {
               "targetBlank": true,
               "title": "Profiling",
-              "url": "https://docs.victoriametrics.com/vmagent.html#profiling"
+              "url": "https://docs.victoriametrics.com/vmagent/#profiling"
             }
           ],
           "options": {
@@ -1691,6 +1723,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -1704,6 +1737,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -1735,7 +1769,8 @@
                   }
                 ]
               },
-              "unit": "short"
+              "unit": "short",
+              "unitScale": true
             },
             "overrides": []
           },
@@ -1743,14 +1778,14 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 41
+            "y": 11
           },
           "id": 56,
           "links": [
             {
               "targetBlank": true,
               "title": "Profiling",
-              "url": "https://docs.victoriametrics.com/vmagent.html#profiling"
+              "url": "https://docs.victoriametrics.com/vmagent/#profiling"
             }
           ],
           "options": {
@@ -1820,6 +1855,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -1833,6 +1869,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -1865,7 +1902,8 @@
                   }
                 ]
               },
-              "unit": "percentunit"
+              "unit": "percentunit",
+              "unitScale": true
             },
             "overrides": []
           },
@@ -1873,7 +1911,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 49
+            "y": 19
           },
           "id": 39,
           "links": [],
@@ -1925,6 +1963,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -1938,6 +1977,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -1970,7 +2010,8 @@
                   }
                 ]
               },
-              "unit": "short"
+              "unit": "short",
+              "unitScale": true
             },
             "overrides": []
           },
@@ -1978,7 +2019,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 49
+            "y": 19
           },
           "id": 41,
           "links": [],
@@ -2017,6 +2058,114 @@
           ],
           "title": "Goroutines ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/troubleshooting/ ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "percentunit",
+              "unitScale": true
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 27
+          },
+          "id": 59,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.2.6",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "max(\n  rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "CPU spent on GC ($instance)",
+          "type": "timeseries"
         }
       ],
       "targets": [
@@ -2107,7 +2256,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 28
+            "y": 36
           },
           "id": 14,
           "options": {
@@ -2209,7 +2358,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 28
+            "y": 36
           },
           "id": 13,
           "options": {
@@ -2311,7 +2460,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 36
+            "y": 44
           },
           "id": 20,
           "options": {
@@ -2414,7 +2563,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 36
+            "y": 44
           },
           "id": 32,
           "options": {
@@ -2513,7 +2662,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 44
+            "y": 52
           },
           "id": 26,
           "options": {
@@ -2583,7 +2732,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows the top $topk recording rules which generate the most of samples. Each generated sample is basically a time series which then ingested into configured remote storage. Rules with high numbers may cause the most pressure on the remote database and become a source of too high cardinality.\n\nThe panel uses MetricsQL functions and may not work with Prometheus.",
+          "description": "Shows the top $topk recording rules which generate the most of [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples). Each generated sample is basically a time series which then ingested into configured remote storage. Rules with high numbers may cause the most pressure on the remote database and become a source of too high cardinality.\n\nThe panel uses MetricsQL functions and may not work with Prometheus.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -2640,7 +2789,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 51
           },
           "id": 31,
           "options": {
@@ -2685,7 +2834,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows the rules which do not produce any samples during the evaluation. Usually it means that such rules are misconfigured, since they give no output during the evaluation.\nPlease check if rule's expression is correct and it is working as expected.",
+          "description": "Shows the rules which do not produce any [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) during the evaluation. Usually it means that such rules are misconfigured, since they give no output during the evaluation.\nPlease check if rule's expression is correct and it is working as expected.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -2742,7 +2891,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 43
+            "y": 51
           },
           "id": 33,
           "options": {
@@ -2843,7 +2992,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 51
+            "y": 59
           },
           "id": 30,
           "options": {
@@ -2964,7 +3113,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 9
+            "y": 17
           },
           "id": 52,
           "options": {
@@ -3056,7 +3205,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 9
+            "y": 17
           },
           "id": 53,
           "options": {
@@ -3086,15 +3235,221 @@
           ],
           "title": "Datapoints drop rate ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Shows current number of established connections to remote write endpoints.\n\n",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 44
+          },
+          "id": 54,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": true,
+              "expr": "sum(max_over_time(vmalert_remotewrite_conns{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)",
+              "interval": "",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Connections ($instance)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Shows the global rate for number of written bytes via remote write connections.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "decbytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 44
+          },
+          "id": 55,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": true,
+              "expr": "sum(rate(vmalert_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0",
+              "interval": "",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Bytes write rate ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Remote write",
       "type": "row"
     }
   ],
-  "refresh": false,
-  "schemaVersion": 37,
-  "style": "dark",
+  "refresh": "",
+  "schemaVersion": 39,
   "tags": [
     "victoriametrics",
     "vmalert"
@@ -3104,8 +3459,8 @@
       {
         "current": {
           "selected": false,
-          "text": "VictoriaMetrics - cluster",
-          "value": "VictoriaMetrics - cluster"
+          "text": "VictoriaMetrics",
+          "value": "P4169E866C3094E38"
         },
         "hide": 0,
         "includeAll": false,
@@ -3171,14 +3526,14 @@
           "type": "prometheus",
           "uid": "$ds"
         },
-        "definition": "label_values(vmalert_iteration_duration_seconds{job=~\"$job\", instance=~\"$instance\"}, group)",
+        "definition": "label_values(vmalert_iteration_total{job=~\"$job\", instance=~\"$instance\"}, group)",
         "hide": 0,
         "includeAll": true,
         "multi": true,
         "name": "group",
         "options": [],
         "query": {
-          "query": "label_values(vmalert_iteration_duration_seconds{job=~\"$job\", instance=~\"$instance\"}, group)",
+          "query": "label_values(vmalert_iteration_total{job=~\"$job\", instance=~\"$instance\"}, group)",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,

hack/download-dashboards.sh

@@ -1,6 +1,6 @@
 #https://github.com/deckhouse/deckhouse/blob/main/modules/340-monitoring-kubernetes-control-plane/monitoring/grafana-dashboards/kubernetes-cluster/control-plane-status.json
 base=https://github.com/deckhouse/deckhouse/raw/main/
-dir="grafana-dashboards"
+dir="dashboards"
 mkdir -p "$dir"
 
 

hack/e2e.sh

@@ -27,9 +27,9 @@ ip link add cozy-br0 type bridge
 ip link set cozy-br0 up
 ip addr add 192.168.123.1/24 dev cozy-br0
 
-# Enable forward & masquerading
-echo 1 > /proc/sys/net/ipv4/ip_forward
-iptables -t nat -A POSTROUTING -s 192.168.123.0/24 -j MASQUERADE
+# Enable masquerading
+iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE 2>/dev/null || true
+iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE
 
 rm -rf srv1 srv2 srv3
 mkdir -p srv1 srv2 srv3
@@ -114,7 +114,7 @@ machine:
     - name: zfs
     - name: spl
   install:
-    image: ghcr.io/aenix-io/cozystack/talos:v1.7.1
+    image: ghcr.io/aenix-io/cozystack/talos:v1.8.0
   files:
   - content: |
       [plugins]
@@ -287,7 +287,8 @@ kubectl patch -n tenant-root hr/tenant-root --type=merge -p '{"spec":{ "values":
   "host": "example.org",
   "ingress": true,
   "monitoring": true,
-  "etcd": true
+  "etcd": true,
+  "isolated": true
 }}}'
 
 # Wait for HelmRelease be created
@@ -296,6 +297,10 @@ timeout 60 sh -c 'until kubectl get hr -n tenant-root etcd ingress monitoring te
 # Wait for HelmReleases be installed
 kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr etcd ingress monitoring tenant-root
 
+kubectl patch -n tenant-root hr/ingress --type=merge -p '{"spec":{ "values":{
+  "dashboard": true
+}}}'
+
 # Wait for nginx-ingress-controller
 timeout 60 sh -c 'until kubectl get deploy -n tenant-root root-ingress-controller; do sleep 1; done'
 kubectl wait --timeout=5m --for=condition=available -n tenant-root deploy root-ingress-controller
@@ -304,8 +309,9 @@ kubectl wait --timeout=5m --for=condition=available -n tenant-root deploy root-i
 kubectl wait --timeout=5m --for=jsonpath=.status.readyReplicas=3 -n tenant-root sts etcd
 
 # Wait for Victoria metrics
-kubectl wait --timeout=5m --for=condition=available deploy -n tenant-root vmalert-vmalert vminsert-longterm vminsert-shortterm
-kubectl wait --timeout=5m --for=jsonpath=.status.readyReplicas=2 -n tenant-root sts vmalertmanager-alertmanager vmselect-longterm vmselect-shortterm vmstorage-longterm vmstorage-shortterm
+kubectl wait --timeout=5m --for=jsonpath=.status.updateStatus=operational -n tenant-root vmalert/vmalert-longterm vmalert/vmalert-shortterm vmalertmanager/alertmanager
+kubectl wait --timeout=5m --for=jsonpath=.status.status=operational -n tenant-root vlogs/generic
+kubectl wait --timeout=5m --for=jsonpath=.status.clusterStatus=operational -n tenant-root vmcluster/shortterm vmcluster/longterm
 
 # Wait for grafana
 kubectl wait --timeout=5m --for=condition=ready -n tenant-root clusters.postgresql.cnpg.io grafana-db

hack/gen_versions_map.sh

@@ -24,24 +24,36 @@ resolved_miss_map=$(
       change_commit=$(git --no-pager blame -L"$line",+1 -- "$chart/Chart.yaml" | awk '{print $1}')
        
       if [ "$change_commit" = "00000000" ]; then
-        # Not commited yet, use previus commit
+        # Not committed yet, use previous commit
         line=$(git show HEAD:"./$chart/Chart.yaml" | awk '/^version:/ {print NR; exit}')
         commit=$(git --no-pager blame -L"$line",+1 HEAD -- "$chart/Chart.yaml" | awk '{print $1}')
         if [ $(echo $commit | cut -c1) = "^" ]; then
-          # Previus commit not exists
+          # Previous commit not exists
           commit=$(echo $commit | cut -c2-)
         fi
       else
-        # Commited, but version_map wasn't updated
+        # Committed, but version_map wasn't updated
         line=$(git show HEAD:"./$chart/Chart.yaml" | awk '/^version:/ {print NR; exit}')
         change_commit=$(git --no-pager blame -L"$line",+1 HEAD -- "$chart/Chart.yaml" | awk '{print $1}')
         if [ $(echo $change_commit | cut -c1) = "^" ]; then
-          # Previus commit not exists
+          # Previous commit not exists
           commit=$(echo $change_commit | cut -c2-)
         else
           commit=$(git describe --always "$change_commit~1")
         fi
       fi
+
+      # Check if the commit belongs to the main branch
+      if ! git merge-base --is-ancestor "$commit" main; then
+        # Find the closest parent commit that belongs to main
+        commit_in_main=$(git log --pretty=format:"%h" main -- "$chart" | head -n 1)
+        if [ -n "$commit_in_main" ]; then
+          commit="$commit_in_main"
+        else
+          # No valid commit found in main branch for $chart, skipping..."
+          continue
+        fi
+      fi
     fi
     echo "$chart $version $commit"
   done

manifests/cozystack-installer.yaml

@@ -68,7 +68,7 @@ spec:
       serviceAccountName: cozystack
       containers:
       - name: cozystack
-        image: "ghcr.io/aenix-io/cozystack/cozystack:v0.10.1"
+        image: "ghcr.io/aenix-io/cozystack/cozystack:v0.16.2"
         env:
         - name: KUBERNETES_SERVICE_HOST
           value: localhost
@@ -87,7 +87,7 @@ spec:
             fieldRef:
               fieldPath: metadata.name
       - name: darkhttpd
-        image: "ghcr.io/aenix-io/cozystack/cozystack:v0.10.1"
+        image: "ghcr.io/aenix-io/cozystack/cozystack:v0.16.2"
         command:
         - /usr/bin/darkhttpd
         - /cozystack/assets

packages/apps/bucket/Makefile

@@ -1,2 +1,4 @@
+include ../../../scripts/package.mk
+
 generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md

packages/apps/clickhouse/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.1
+version: 0.5.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/clickhouse/Makefile

@@ -1,2 +1,20 @@
+CLICKHOUSE_BACKUP_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
+
+include ../../../scripts/common-envs.mk
+include ../../../scripts/package.mk
+
 generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md
+
+image:
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/clickhouse-backup \
+		--provenance false \
+		--tag $(REGISTRY)/clickhouse-backup:$(call settag,$(CLICKHOUSE_BACKUP_TAG)) \
+		--cache-from type=registry,ref=$(REGISTRY)/clickhouse-backup:latest \
+		--cache-to type=inline \
+		--metadata-file images/clickhouse-backup.json \
+		--push=$(PUSH) \
+		--load=$(LOAD)
+	echo "$(REGISTRY)/clickhouse-backup:$(call settag,$(CLICKHOUSE_BACKUP_TAG))@$$(yq e '."containerimage.digest"' images/clickhouse-backup.json -o json -r)" \
+		> images/clickhouse-backup.tag
+	rm -f images/clickhouse-backup.json

packages/apps/clickhouse/README.md

@@ -1,17 +1,46 @@
 # Managed Clickhouse Service
 
+### How to restore backup:
+
+find snapshot:
+```
+restic -r s3:s3.example.org/clickhouse-backups/table_name snapshots
+```
+
+restore:
+```
+restic -r s3:s3.example.org/clickhouse-backups/table_name restore latest --target /tmp/
+```
+
+more details:
+- https://itnext.io/restic-effective-backup-from-stdin-4bc1e8f083c1
+
 ## Parameters
 
 ### Common parameters
 
-| Name       | Description                   | Value  |
-| ---------- | ----------------------------- | ------ |
-| `size`     | Persistent Volume size        | `10Gi` |
-| `shards`   | Number of Clickhouse replicas | `1`    |
-| `replicas` | Number of Clickhouse shards   | `2`    |
+| Name           | Description                         | Value  |
+| -------------- | ----------------------------------- | ------ |
+| `size`         | Persistent Volume size              | `10Gi` |
+| `shards`       | Number of Clickhouse replicas       | `1`    |
+| `replicas`     | Number of Clickhouse shards         | `2`    |
+| `storageClass` | StorageClass used to store the data | `""`   |
 
 ### Configuration parameters
 
 | Name    | Description         | Value |
 | ------- | ------------------- | ----- |
 | `users` | Users configuration | `{}`  |
+
+### Backup parameters
+
+| Name                     | Description                                    | Value                                                  |
+| ------------------------ | ---------------------------------------------- | ------------------------------------------------------ |
+| `backup.enabled`         | Enable pereiodic backups                       | `false`                                                |
+| `backup.s3Region`        | The AWS S3 region where backups are stored     | `us-east-1`                                            |
+| `backup.s3Bucket`        | The S3 bucket used for storing backups         | `s3.example.org/clickhouse-backups`                    |
+| `backup.schedule`        | Cron schedule for automated backups            | `0 2 * * *`                                            |
+| `backup.cleanupStrategy` | The strategy for cleaning up old backups       | `--keep-last=3 --keep-daily=3 --keep-within-weekly=1m` |
+| `backup.s3AccessKey`     | The access key for S3, used for authentication | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu`                     |
+| `backup.s3SecretKey`     | The secret key for S3, used for authentication | `ju3eum4dekeich9ahM1te8waeGai0oog`                     |
+| `backup.resticPassword`  | The password for Restic backup encryption      | `ChaXoveekoh6eigh4siesheeda2quai0`                     |

packages/apps/clickhouse/images/clickhouse-backup.tag

@@ -0,0 +1 @@
+ghcr.io/aenix-io/cozystack/clickhouse-backup:0.5.0@sha256:dda84420cb8648721299221268a00d72a05c7af5b7fb452619bac727068b9e61

packages/apps/clickhouse/images/clickhouse-backup/Dockerfile

@@ -0,0 +1,2 @@
+FROM clickhouse/clickhouse-server:24.8.4-alpine
+RUN apk add --no-cache restic uuidgen

packages/apps/clickhouse/templates/backup-cronjob.yaml

@@ -0,0 +1,95 @@
+{{- if .Values.backup.enabled }}
+{{ $image := .Files.Get "images/backup.json" | fromJson }}
+
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ .Release.Name }}-backup
+spec:
+  schedule: "{{ .Values.backup.schedule }}"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: OnFailure
+      template:
+        metadata:
+          annotations:
+            checksum/config: {{ include (print $.Template.BasePath "/backup-script.yaml") . | sha256sum }}
+            checksum/secret: {{ include (print $.Template.BasePath "/backup-secret.yaml") . | sha256sum }}
+        spec:
+          imagePullSecrets:
+          - name: {{ .Release.Name }}-regsecret
+          restartPolicy: Never
+          containers:
+          - name: clickhouse-backup
+            image: "{{ $.Files.Get "images/clickhouse-backup.tag" | trim }}"
+            command:
+            - /bin/sh
+            - -x
+            - /scripts/backup.sh
+            env:
+            - name: REPO_PREFIX
+              value: {{ required "s3Bucket is not specified!" .Values.backup.s3Bucket | quote }}
+            - name: CLEANUP_STRATEGY
+              value: {{ required "cleanupPolicy is not specified!" .Values.backup.cleanupStrategy | quote }}
+            - name: CLICKHOUSE_USER
+              value: backup
+            - name: CLICKHOUSE_HOST
+              value: chi-{{ .Release.Name }}-clickhouse-0-0
+            - name: CLICKHOUSE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-credentials
+                  key: backup
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-backup
+                  key: s3AccessKey
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-backup
+                  key: s3SecretKey
+            - name: AWS_DEFAULT_REGION
+              value: {{ .Values.backup.s3Region }}
+            - name: RESTIC_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-backup
+                  key: resticPassword
+            volumeMounts:
+            - mountPath: /scripts
+              name: scripts
+            - mountPath: /tmp
+              name: tmp
+            - mountPath: /.cache
+              name: cache
+            securityContext:
+              allowPrivilegeEscalation: false
+              capabilities:
+                drop:
+                - ALL
+              privileged: false
+              readOnlyRootFilesystem: true
+              runAsNonRoot: true
+          volumes:
+          - name: scripts
+            secret:
+              secretName: {{ .Release.Name }}-backup-script
+          - name: tmp
+            emptyDir: {}
+          - name: cache
+            emptyDir: {}
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 9000
+            runAsGroup: 9000
+            seccompProfile:
+              type: RuntimeDefault
+{{- end }}

packages/apps/clickhouse/templates/backup-script.yaml

@@ -0,0 +1,55 @@
+{{- if .Values.backup.enabled }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-backup-script
+stringData:
+  backup.sh: |
+    #!/bin/sh
+    set -e
+    set -o pipefail
+    
+    JOB_ID="job-$(uuidgen|cut -f1 -d-)"
+    TABLE_LIST=$(clickhouse-client --host "$CLICKHOUSE_HOST" -q 'SHOW TABLES;' | grep -v '^.inner.' || true)
+    echo DB_LIST=$(echo "$TABLE_LIST" | shuf) # shuffle list
+    echo "Job ID: $JOB_ID"
+    echo "Target repo: $REPO_PREFIX"
+    echo "Cleanup strategy: $CLEANUP_STRATEGY"
+    echo "Start backup for:"
+    echo "$TABLE_LIST"
+    echo
+    echo "Backup started at `date +%Y-%m-%d\ %H:%M:%S`"
+    for table in $TABLE_LIST; do
+      (
+        set -x
+        restic -r "s3:${REPO_PREFIX}/$table" cat config >/dev/null 2>&1 || \
+          restic -r "s3:${REPO_PREFIX}/$table" init --repository-version 2
+        restic -r "s3:${REPO_PREFIX}/$table" unlock --remove-all >/dev/null 2>&1 || true # no locks, k8s takes care of it
+        clickhouse-client --host "$CLICKHOUSE_HOST" -q "SHOW CREATE TABLE ${table}" | awk '{gsub(/\\n/, "\n")} {gsub(/\\'\''/, "'\''")} 1' | \
+          restic -r "s3:${REPO_PREFIX}/$table" backup --tag "$JOB_ID" --tag index --stdin --stdin-filename index.sql
+        clickhouse-client --host "$CLICKHOUSE_HOST" -q "SELECT * FROM ${table} FORMAT TabSeparated" | \
+          restic -r "s3:${REPO_PREFIX}/$table" backup --tag "$JOB_ID" --tag data --stdin --stdin-filename data.tsv
+        restic -r "s3:${REPO_PREFIX}/$table" tag --tag "$JOB_ID" --set "completed"
+      )
+    done
+    echo "Backup finished at `date +%Y-%m-%d\ %H:%M:%S`"
+    
+    echo
+    echo "Run cleanup:"
+    echo
+    
+    echo "Cleanup started at `date +%Y-%m-%d\ %H:%M:%S`"
+    for db in $DB_LIST; do
+      (
+        set -x
+        # keep completed snapshots only
+        restic forget -r "s3:${REPO_PREFIX}/$db" --group-by=tags --keep-tag "completed" --tag index
+        restic forget -r "s3:${REPO_PREFIX}/$db" --group-by=tags --keep-tag "completed" --tag data
+        restic forget -r "s3:${REPO_PREFIX}/$db" --group-by=tags $CLEANUP_STRATEGY --tag index
+        restic forget -r "s3:${REPO_PREFIX}/$db" --group-by=tags $CLEANUP_STRATEGY --tag data
+        restic prune -r "s3:${REPO_PREFIX}/$db"
+      )
+    done
+    echo "Cleanup finished at `date +%Y-%m-%d\ %H:%M:%S`"
+{{- end }}

packages/apps/clickhouse/templates/backup-secret.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.backup.enabled }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-backup
+stringData:
+  s3AccessKey: {{ required "s3AccessKey is not specified!" .Values.backup.s3AccessKey }}
+  s3SecretKey: {{ required "s3SecretKey is not specified!" .Values.backup.s3SecretKey }}
+  resticPassword: {{ required "resticPassword is not specified!" .Values.backup.resticPassword }}
+{{- end }}

packages/apps/clickhouse/templates/clickhouse.yaml

@@ -1,3 +1,32 @@
+{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (printf "%s-credentials" .Release.Name) }}
+{{- $passwords := dict }}
+{{- $users := .Values.users }}
+{{- $_ := set $users "backup" dict }}
+
+{{- with (index $existingSecret "data") }}
+  {{- range $k, $v := . }}
+    {{- $_ := set $passwords $k (b64dec $v) }}
+  {{- end }}
+{{- end }}
+
+{{- range $user, $u := $users }}
+  {{- if $u.password }}
+    {{- $_ := set $passwords $user $u.password }}
+  {{- else if not (index $passwords $user) }}
+    {{- $_ := set $passwords $user (randAlphaNum 16) }}
+  {{- end }}
+{{- end }}
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-credentials
+stringData:
+  {{- range $user, $u := .Values.users }}
+  {{ quote $user }}: {{ quote (index $passwords $user) }}
+  {{- end }}
+
+---
 apiVersion: "clickhouse.altinity.com/v1"
 kind: "ClickHouseInstallation"
 metadata:
@@ -9,10 +38,10 @@ spec:
       dataVolumeClaimTemplate: data-volume-template
   {{- end }}
   configuration:
-    {{- with .Values.users }}
+    {{- with $users }}
     users:
       {{- range $name, $u := . }}
-      {{ $name }}/password_sha256_hex: {{ sha256sum $u.password }}
+      {{ $name }}/password_sha256_hex: {{ sha256sum (index $passwords $name) }}
       {{ $name }}/profile: {{ ternary "readonly" "default" (index $u "readonly" | default false) }}
       {{ $name }}/networks/ip: ["::/0"]
       {{- end }}
@@ -31,6 +60,9 @@ spec:
         spec:
           accessModes:
             - ReadWriteOnce
+          {{- with $.Values.storageClass }}
+          storageClassName: {{ . }}
+          {{- end }}
           resources:
             requests:
               storage: {{ . }}

packages/apps/clickhouse/templates/dashboard-resourcemap.yaml

@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}-dashboard-resources
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  resourceNames:
+  - chi-clickhouse-test-clickhouse-0-0
+  verbs: ["get", "list", "watch"]
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  resourceNames:
+  - {{ .Release.Name }}-credentials
+  verbs: ["get", "list", "watch"]

packages/apps/clickhouse/values.schema.json

@@ -16,6 +16,56 @@
             "type": "number",
             "description": "Number of Clickhouse shards",
             "default": 2
+        },
+        "storageClass": {
+            "type": "string",
+            "description": "StorageClass used to store the data",
+            "default": ""
+        },
+        "backup": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "boolean",
+                    "description": "Enable pereiodic backups",
+                    "default": false
+                },
+                "s3Region": {
+                    "type": "string",
+                    "description": "The AWS S3 region where backups are stored",
+                    "default": "us-east-1"
+                },
+                "s3Bucket": {
+                    "type": "string",
+                    "description": "The S3 bucket used for storing backups",
+                    "default": "s3.example.org/clickhouse-backups"
+                },
+                "schedule": {
+                    "type": "string",
+                    "description": "Cron schedule for automated backups",
+                    "default": "0 2 * * *"
+                },
+                "cleanupStrategy": {
+                    "type": "string",
+                    "description": "The strategy for cleaning up old backups",
+                    "default": "--keep-last=3 --keep-daily=3 --keep-within-weekly=1m"
+                },
+                "s3AccessKey": {
+                    "type": "string",
+                    "description": "The access key for S3, used for authentication",
+                    "default": "oobaiRus9pah8PhohL1ThaeTa4UVa7gu"
+                },
+                "s3SecretKey": {
+                    "type": "string",
+                    "description": "The secret key for S3, used for authentication",
+                    "default": "ju3eum4dekeich9ahM1te8waeGai0oog"
+                },
+                "resticPassword": {
+                    "type": "string",
+                    "description": "The password for Restic backup encryption",
+                    "default": "ChaXoveekoh6eigh4siesheeda2quai0"
+                }
+            }
         }
     }
 }

packages/apps/clickhouse/values.yaml

@@ -3,10 +3,12 @@
 ## @param size Persistent Volume size
 ## @param shards Number of Clickhouse replicas
 ## @param replicas Number of Clickhouse shards
+## @param storageClass StorageClass used to store the data
 ##
 size: 10Gi
 shards: 1
 replicas: 2
+storageClass: ""
 
 ## @section Configuration parameters
 
@@ -20,3 +22,23 @@ replicas: 2
 ##     password: hackme
 ##
 users: {}
+
+## @section Backup parameters
+
+## @param backup.enabled Enable pereiodic backups
+## @param backup.s3Region The AWS S3 region where backups are stored
+## @param backup.s3Bucket The S3 bucket used for storing backups
+## @param backup.schedule Cron schedule for automated backups
+## @param backup.cleanupStrategy The strategy for cleaning up old backups
+## @param backup.s3AccessKey The access key for S3, used for authentication
+## @param backup.s3SecretKey The secret key for S3, used for authentication
+## @param backup.resticPassword The password for Restic backup encryption
+backup:
+  enabled: false
+  s3Region: us-east-1
+  s3Bucket: s3.example.org/clickhouse-backups
+  schedule: "0 2 * * *"
+  cleanupStrategy: "--keep-last=3 --keep-daily=3 --keep-within-weekly=1m"
+  s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
+  s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
+  resticPassword: ChaXoveekoh6eigh4siesheeda2quai0

packages/apps/ferretdb/Chart.yaml

@@ -16,10 +16,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.4.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.22.0"
+appVersion: "1.24.0"

packages/apps/ferretdb/Makefile

@@ -1,2 +1,4 @@
+include ../../../scripts/package.mk
+
 generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md

packages/apps/ferretdb/README.md

@@ -9,6 +9,7 @@
 | `external`               | Enable external access from outside the cluster                                                                         | `false` |
 | `size`                   | Persistent Volume size                                                                                                  | `10Gi`  |
 | `replicas`               | Number of Postgres replicas                                                                                             | `2`     |
+| `storageClass`           | StorageClass used to store the data                                                                                     | `""`    |
 | `quorum.minSyncReplicas` | Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.           | `0`     |
 | `quorum.maxSyncReplicas` | Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances). | `0`     |
 

packages/apps/ferretdb/images/postgres-backup.tag

@@ -0,0 +1 @@
+ghcr.io/aenix-io/cozystack/postgres-backup:0.7.0@sha256:d2015c6dba92293bda652d055e97d1be80e8414c2dc78037c12812d1a2e2cba1

packages/apps/ferretdb/templates/backup-cronjob.yaml

@@ -24,8 +24,8 @@ spec:
         spec:
           restartPolicy: Never
           containers:
-          - name: mysqldump
-            image: "{{ index $image "image.name" }}@{{ index $image "containerimage.digest" }}"
+          - name: pgdump
+            image: "{{ $.Files.Get "images/postgres-backup.tag" | trim }}"
             command:
             - /bin/sh
             - /scripts/backup.sh

packages/apps/ferretdb/templates/dashboard-resourcemap.yaml

@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}-dashboard-resources
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  resourceNames:
+  - {{ .Release.Name }}
+  verbs: ["get", "list", "watch"]
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  resourceNames:
+  - {{ .Release.Name }}-credentials
+  verbs: ["get", "list", "watch"]

packages/apps/ferretdb/templates/ferretdb.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: ferretdb
-        image: ghcr.io/ferretdb/ferretdb:1.22.0
+        image: ghcr.io/ferretdb/ferretdb:1.24.0
         ports:
         - containerPort: 27017
         env:

packages/apps/ferretdb/templates/init-script.yaml

@@ -1,3 +1,30 @@
+{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (printf "%s-credentials" .Release.Name) }}
+{{- $passwords := dict }}
+
+{{- with (index $existingSecret "data") }}
+  {{- range $k, $v := . }}
+    {{- $_ := set $passwords $k (b64dec $v) }}
+  {{- end }}
+{{- end }}
+
+{{- range $user, $u := .Values.users }}
+  {{- if $u.password }}
+    {{- $_ := set $passwords $user $u.password }}
+  {{- else if not (index $passwords $user) }}
+    {{- $_ := set $passwords $user (randAlphaNum 16) }}
+  {{- end }}
+{{- end }}
+
+{{- if .Values.users }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-credentials
+stringData:
+  {{- range $user, $u := .Values.users }}
+  {{ quote $user }}: {{ quote (index $passwords $user) }}
+  {{- end }}
+{{- end }}
 ---
 apiVersion: v1
 kind: Secret
@@ -13,7 +40,7 @@ stringData:
     {{- range $user, $u := .Values.users }}
     SELECT 'CREATE ROLE {{ $user }} LOGIN INHERIT;'
     WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{{ $user }}')\gexec
-    ALTER ROLE {{ $user }} WITH PASSWORD '{{ $u.password }}' LOGIN INHERIT {{ ternary "REPLICATION" "NOREPLICATION" (default false $u.replication) }};
+    ALTER ROLE {{ $user }} WITH PASSWORD '{{ index $passwords $user }}' LOGIN INHERIT {{ ternary "REPLICATION" "NOREPLICATION" (default false $u.replication) }};
     COMMENT ON ROLE {{ $user }} IS 'user managed by helm';
     {{- end }}
     EOT

packages/apps/ferretdb/templates/postgres.yaml

@@ -15,6 +15,9 @@ spec:
 
   storage:
     size: {{ required ".Values.size is required" .Values.size }}
+    {{- with .Values.storageClass }}
+    storageClass: {{ . }}
+    {{- end }}
 
   inheritedMetadata:
     labels:

packages/apps/ferretdb/values.schema.json

@@ -17,6 +17,11 @@
             "description": "Number of Postgres replicas",
             "default": 2
         },
+        "storageClass": {
+            "type": "string",
+            "description": "StorageClass used to store the data",
+            "default": ""
+        },
         "quorum": {
             "type": "object",
             "properties": {

packages/apps/ferretdb/values.yaml

@@ -3,10 +3,12 @@
 ## @param external Enable external access from outside the cluster
 ## @param size Persistent Volume size
 ## @param replicas Number of Postgres replicas
+## @param storageClass StorageClass used to store the data
 ##
 external: false
 size: 10Gi
 replicas: 2
+storageClass: ""
 
 ## Configuration for the quorum-based synchronous replication
 ## @param quorum.minSyncReplicas Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.

packages/apps/ferretdb/values2.yaml

@@ -1,56 +0,0 @@
-## @section Common parameters
-
-## @param external Enable external access from outside the cluster
-## @param size Persistent Volume size
-## @param replicas Number of Postgres replicas
-##
-external: false
-size: 10Gi
-replicas: 1
-
-## Configuration for the quorum-based synchronous replication
-## @param quorum.minSyncReplicas Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.
-## @param quorum.maxSyncReplicas Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).
-quorum:
-  minSyncReplicas: 0
-  maxSyncReplicas: 0
-
-## @section Configuration parameters
-
-## @param users [object] Users configuration
-## Example:
-## users:
-##   user1:
-##     password: strongpassword
-##   user2:
-##     password: hackme
-##
-users:
-  foo:
-    password: asd
-  bar:
-    password: asd
-  baz:
-    password: asd
-  boo:
-    password: asd
-
-## @section Backup parameters
-
-## @param backup.enabled Enable pereiodic backups
-## @param backup.s3Region The AWS S3 region where backups are stored
-## @param backup.s3Bucket The S3 bucket used for storing backups
-## @param backup.schedule Cron schedule for automated backups
-## @param backup.cleanupStrategy The strategy for cleaning up old backups
-## @param backup.s3AccessKey The access key for S3, used for authentication
-## @param backup.s3SecretKey The secret key for S3, used for authentication
-## @param backup.resticPassword The password for Restic backup encryption
-backup:
-  enabled: false
-  s3Region: us-east-1
-  s3Bucket: s3.example.org/postgres-backups
-  schedule: "0 2 * * *"
-  cleanupStrategy: "--keep-last=3 --keep-daily=3 --keep-within-weekly=1m"
-  s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
-  s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
-  resticPassword: ChaXoveekoh6eigh4siesheeda2quai0

packages/apps/http-cache/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: http-cache
-description: Layer7 load balacner and caching service
+description: Layer7 load balancer and caching service
 icon: /logos/nginx.svg
 
 # A chart can be either an 'application' or a 'library' chart.
@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.0
+version: 0.3.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/http-cache/Makefile

@@ -1,6 +1,7 @@
-NGINX_CACHE_TAG = v0.1.0
+NGINX_CACHE_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
 
 include ../../../scripts/common-envs.mk
+include ../../../scripts/package.mk
 
 image: image-nginx
 
@@ -8,7 +9,6 @@ image-nginx:
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/nginx-cache \
 		--provenance false \
 		--tag $(REGISTRY)/nginx-cache:$(call settag,$(NGINX_CACHE_TAG)) \
-		--tag $(REGISTRY)/nginx-cache:$(call settag,$(NGINX_CACHE_TAG)-$(TAG)) \
 		--cache-from type=registry,ref=$(REGISTRY)/nginx-cache:latest \
 		--cache-to type=inline \
 		--metadata-file images/nginx-cache.json \

packages/apps/http-cache/README.md

@@ -64,6 +64,7 @@ VTS module shows wrong upstream resonse time
 | ------------------ | ----------------------------------------------- | ------- |
 | `external`         | Enable external access from outside the cluster | `false` |
 | `size`             | Persistent Volume size                          | `10Gi`  |
+| `storageClass`     | StorageClass used to store the data             | `""`    |
 | `haproxy.replicas` | Number of HAProxy replicas                      | `2`     |
 | `nginx.replicas`   | Number of Nginx replicas                        | `2`     |
 

packages/apps/http-cache/images/nginx-cache.tag

@@ -1 +1 @@
-ghcr.io/aenix-io/cozystack/nginx-cache:v0.1.0@sha256:f77d5b63f1ed9dfda4725696d9170130939219a2465260b6ba941947460de2da
+ghcr.io/aenix-io/cozystack/nginx-cache:0.3.1@sha256:cd744b2d1d50191f4908f2db83079b32973d1c009fe9468627be72efbfa0a107

packages/apps/http-cache/templates/nginx/deployment.yaml

@@ -114,6 +114,9 @@ spec:
   resources:
     requests:
       storage: "{{ $.Values.size }}"
+  {{- with $.Values.storageClass }}
+  storageClassName: {{ . }}
+  {{- end }}
 ---
 apiVersion: v1
 kind: Service

packages/apps/http-cache/values.schema.json

@@ -12,6 +12,11 @@
             "description": "Persistent Volume size",
             "default": "10Gi"
         },
+        "storageClass": {
+            "type": "string",
+            "description": "StorageClass used to store the data",
+            "default": ""
+        },
         "haproxy": {
             "type": "object",
             "properties": {

packages/apps/http-cache/values.yaml

@@ -3,11 +3,13 @@
 
 ## @param external Enable external access from outside the cluster
 ## @param size Persistent Volume size
+## @param storageClass StorageClass used to store the data
 ## @param haproxy.replicas Number of HAProxy replicas
 ## @param nginx.replicas Number of Nginx replicas
 ##
 external: false
 size: 10Gi
+storageClass: ""
 haproxy:
   replicas: 2
 nginx:

packages/apps/kafka/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.2
+version: 0.3.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/kafka/Makefile

@@ -1,2 +1,4 @@
+include ../../../scripts/package.mk
+
 generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md

packages/apps/kafka/README.md

@@ -4,13 +4,15 @@
 
 ### Common parameters
 
-| Name                 | Description                                     | Value   |
-| -------------------- | ----------------------------------------------- | ------- |
-| `external`           | Enable external access from outside the cluster | `false` |
-| `kafka.size`         | Persistent Volume size for Kafka                | `10Gi`  |
-| `kafka.replicas`     | Number of Kafka replicas                        | `3`     |
-| `zookeeper.size`     | Persistent Volume size for ZooKeeper            | `5Gi`   |
-| `zookeeper.replicas` | Number of ZooKeeper replicas                    | `3`     |
+| Name                     | Description                                     | Value   |
+| ------------------------ | ----------------------------------------------- | ------- |
+| `external`               | Enable external access from outside the cluster | `false` |
+| `kafka.size`             | Persistent Volume size for Kafka                | `10Gi`  |
+| `kafka.replicas`         | Number of Kafka replicas                        | `3`     |
+| `kafka.storageClass`     | StorageClass used to store the Kafka data       | `""`    |
+| `zookeeper.size`         | Persistent Volume size for ZooKeeper            | `5Gi`   |
+| `zookeeper.replicas`     | Number of ZooKeeper replicas                    | `3`     |
+| `zookeeper.storageClass` | StorageClass used to store the ZooKeeper data   | `""`    |
 
 ### Configuration parameters
 

packages/apps/kafka/templates/kafka.yaml

@@ -53,6 +53,9 @@ spec:
         {{- with .Values.kafka.size }}
         size: {{ . }}
         {{- end }}
+        {{- with .Values.kafka.storageClass }}
+        class: {{ . }}
+        {{- end }}
         deleteClaim: true
   zookeeper:
     replicas: {{ .Values.zookeeper.replicas }}
@@ -61,6 +64,9 @@ spec:
       {{- with .Values.zookeeper.size }}
       size: {{ . }}
       {{- end }}
+      {{- with .Values.kafka.storageClass }}
+      class: {{ . }}
+      {{- end }}
       deleteClaim: false
   entityOperator:
     topicOperator: {}

packages/apps/kafka/values.schema.json

@@ -19,6 +19,11 @@
                     "type": "number",
                     "description": "Number of Kafka replicas",
                     "default": 3
+                },
+                "storageClass": {
+                    "type": "string",
+                    "description": "StorageClass used to store the Kafka data",
+                    "default": ""
                 }
             }
         },
@@ -34,6 +39,11 @@
                     "type": "number",
                     "description": "Number of ZooKeeper replicas",
                     "default": 3
+                },
+                "storageClass": {
+                    "type": "string",
+                    "description": "StorageClass used to store the ZooKeeper data",
+                    "default": ""
                 }
             }
         },

packages/apps/kafka/values.yaml

@@ -4,16 +4,20 @@
 ## @param external Enable external access from outside the cluster
 ## @param kafka.size Persistent Volume size for Kafka
 ## @param kafka.replicas Number of Kafka replicas
+## @param kafka.storageClass StorageClass used to store the Kafka data
 ## @param zookeeper.size Persistent Volume size for ZooKeeper
 ## @param zookeeper.replicas Number of ZooKeeper replicas
+## @param zookeeper.storageClass StorageClass used to store the ZooKeeper data
 ##
 external: false
 kafka:
   size: 10Gi
   replicas: 3
+  storageClass: ""
 zookeeper:
   size: 5Gi
   replicas: 3
+  storageClass: ""
 
 ## @section Configuration parameters
 

packages/apps/kubernetes/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.8.2
+version: 0.12.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/kubernetes/Makefile

@@ -1,11 +1,13 @@
 UBUNTU_CONTAINER_DISK_TAG = v1.30.1
+KUBERNETES_PKG_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
 
 include ../../../scripts/common-envs.mk
+include ../../../scripts/package.mk
 
 generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md
 
-image: image-ubuntu-container-disk
+image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler
 
 image-ubuntu-container-disk:
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/ubuntu-container-disk \
@@ -20,3 +22,46 @@ image-ubuntu-container-disk:
 	echo "$(REGISTRY)/ubuntu-container-disk:$(call settag,$(UBUNTU_CONTAINER_DISK_TAG))@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk.json -o json -r)" \
 		> images/ubuntu-container-disk.tag
 	rm -f images/ubuntu-container-disk.json
+
+image-kubevirt-cloud-provider:
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/kubevirt-cloud-provider \
+		--provenance false \
+		--tag $(REGISTRY)/kubevirt-cloud-provider:$(call settag,$(KUBERNETES_PKG_TAG)) \
+		--tag $(REGISTRY)/kubevirt-cloud-provider:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \
+		--cache-from type=registry,ref=$(REGISTRY)/kubevirt-cloud-provider:latest \
+		--cache-to type=inline \
+		--metadata-file images/kubevirt-cloud-provider.json \
+		--push=$(PUSH) \
+		--load=$(LOAD)
+	echo "$(REGISTRY)/kubevirt-cloud-provider:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/kubevirt-cloud-provider.json -o json -r)" \
+		> images/kubevirt-cloud-provider.tag
+	rm -f images/kubevirt-cloud-provider.json
+
+image-kubevirt-csi-driver:
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/kubevirt-csi-driver \
+		--provenance false \
+		--tag $(REGISTRY)/kubevirt-csi-driver:$(call settag,$(KUBERNETES_PKG_TAG)) \
+		--tag $(REGISTRY)/kubevirt-csi-driver:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \
+		--cache-from type=registry,ref=$(REGISTRY)/kubevirt-csi-driver:latest \
+		--cache-to type=inline \
+		--metadata-file images/kubevirt-csi-driver.json \
+		--push=$(PUSH) \
+		--load=$(LOAD)
+	echo "$(REGISTRY)/kubevirt-csi-driver:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/kubevirt-csi-driver.json -o json -r)" \
+		> images/kubevirt-csi-driver.tag
+	rm -f images/kubevirt-csi-driver.json
+
+
+image-cluster-autoscaler:
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/cluster-autoscaler \
+		--provenance false \
+		--tag $(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG)) \
+		--tag $(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \
+		--cache-from type=registry,ref=$(REGISTRY)/cluster-autoscaler:latest \
+		--cache-to type=inline \
+		--metadata-file images/cluster-autoscaler.json \
+		--push=$(PUSH) \
+		--load=$(LOAD)
+	echo "$(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/cluster-autoscaler.json -o json -r)" \
+		> images/cluster-autoscaler.tag
+	rm -f images/cluster-autoscaler.json

packages/apps/kubernetes/README.md

@@ -31,18 +31,22 @@ kubectl get secret -n <namespace> kubernetes-<clusterName>-admin-kubeconfig -o g
 
 ### Common parameters
 
-| Name                    | Description                                                                                                                            | Value |
-| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `host`                  | The hostname used to access the Kubernetes cluster externally (defaults to using the cluster name as a subdomain for the tenant host). | `""`  |
-| `controlPlane.replicas` | Number of replicas for Kubernetes contorl-plane components                                                                             | `2`   |
-| `nodeGroups`            | nodeGroups configuration                                                                                                               | `{}`  |
+| Name                    | Description                                                                                                                            | Value        |
+| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| `host`                  | The hostname used to access the Kubernetes cluster externally (defaults to using the cluster name as a subdomain for the tenant host). | `""`         |
+| `controlPlane.replicas` | Number of replicas for Kubernetes contorl-plane components                                                                             | `2`          |
+| `storageClass`          | StorageClass used to store user data                                                                                                   | `replicated` |
+| `nodeGroups`            | nodeGroups configuration                                                                                                               | `{}`         |
 
 ### Cluster Addons
 
-| Name                          | Description                                                                        | Value   |
-| ----------------------------- | ---------------------------------------------------------------------------------- | ------- |
-| `addons.certManager.enabled`  | Enables the cert-manager                                                           | `false` |
-| `addons.ingressNginx.enabled` | Enable Ingress-NGINX controller (expect nodes with 'ingress-nginx' role)           | `false` |
-| `addons.ingressNginx.hosts`   | List of domain names that should be passed through to the cluster by upper cluster | `[]`    |
-| `addons.fluxcd.enabled`       | Enables Flux CD                                                                    | `false` |
+| Name                                 | Description                                                                        | Value   |
+| ------------------------------------ | ---------------------------------------------------------------------------------- | ------- |
+| `addons.certManager.enabled`         | Enables the cert-manager                                                           | `false` |
+| `addons.certManager.valuesOverride`  | Custom values to override                                                          | `{}`    |
+| `addons.ingressNginx.enabled`        | Enable Ingress-NGINX controller (expect nodes with 'ingress-nginx' role)           | `false` |
+| `addons.ingressNginx.valuesOverride` | Custom values to override                                                          | `{}`    |
+| `addons.ingressNginx.hosts`          | List of domain names that should be passed through to the cluster by upper cluster | `[]`    |
+| `addons.fluxcd.enabled`              | Enables Flux CD                                                                    | `false` |
+| `addons.fluxcd.valuesOverride`       | Custom values to override                                                          | `{}`    |
 

packages/apps/kubernetes/images/cluster-autoscaler.tag

@@ -0,0 +1 @@
+ghcr.io/aenix-io/cozystack/cluster-autoscaler:0.11.1@sha256:7f617de5a24de790a15d9e97c6287ff2b390922e6e74c7a665cbf498f634514d

packages/apps/kubernetes/images/cluster-autoscaler/Dockerfile

@@ -0,0 +1,17 @@
+# Source: https://raw.githubusercontent.com/kubernetes/autoscaler/refs/heads/master/cluster-autoscaler/Dockerfile.amd64
+ARG builder_image=docker.io/library/golang:1.22.5
+ARG BASEIMAGE=gcr.io/distroless/static:nonroot-amd64
+FROM ${builder_image} AS builder
+RUN git clone https://github.com/kubernetes/autoscaler /src/autoscaler \
+ && cd /src/autoscaler/cluster-autoscaler \
+ && git checkout cluster-autoscaler-1.31.0
+
+WORKDIR /src/autoscaler/cluster-autoscaler
+RUN make build
+
+FROM $BASEIMAGE
+LABEL maintainer="Marcin Wielgus <mwielgus@google.com>"
+
+COPY --from=builder /src/autoscaler/cluster-autoscaler/cluster-autoscaler-amd64 /cluster-autoscaler
+WORKDIR /
+CMD ["/cluster-autoscaler"]

packages/apps/kubernetes/images/kubevirt-cloud-provider.tag

@@ -0,0 +1 @@
+ghcr.io/aenix-io/cozystack/kubevirt-cloud-provider:0.11.1@sha256:b5aa62a53be566b49dea635ce8f6b9280566e260f8493ff3d71f8c7501fb4cbc

packages/apps/kubernetes/images/kubevirt-cloud-provider/Dockerfile

@@ -0,0 +1,21 @@
+# Source: https://github.com/kubevirt/cloud-provider-kubevirt/blob/main/build/images/kubevirt-cloud-controller-manager/Dockerfile
+FROM --platform=linux/amd64 golang:1.20.6 AS builder
+
+RUN git clone https://github.com/kubevirt/cloud-provider-kubevirt /go/src/kubevirt.io/cloud-provider-kubevirt \
+ && cd /go/src/kubevirt.io/cloud-provider-kubevirt \
+ && git checkout adbd6c27468b86b020cf38490e84f124ef24ab62
+
+WORKDIR /go/src/kubevirt.io/cloud-provider-kubevirt
+
+# see: https://github.com/kubevirt/cloud-provider-kubevirt/pull/291
+ADD patches /patches
+RUN git apply /patches/external-traffic-policy-local.diff
+RUN go get 'k8s.io/endpointslice/util@v0.28' 'k8s.io/apiserver@v0.28'
+RUN go mod tidy
+RUN go mod vendor
+
+RUN	CGO_ENABLED=0 GOOS=linux go build -mod=vendor -ldflags="-s -w" -o bin/kubevirt-cloud-controller-manager ./cmd/kubevirt-cloud-controller-manager
+
+FROM registry.access.redhat.com/ubi9/ubi-micro
+COPY --from=builder /go/src/kubevirt.io/cloud-provider-kubevirt/bin/kubevirt-cloud-controller-manager /bin/kubevirt-cloud-controller-manager
+ENTRYPOINT [ "/bin/kubevirt-cloud-controller-manager" ]

packages/apps/kubernetes/images/kubevirt-csi-driver.tag

@@ -0,0 +1 @@
+ghcr.io/aenix-io/cozystack/kubevirt-csi-driver:0.11.1@sha256:705e20e638315501aaa8b8156ceb8b260086b21876aa994bec9d6c406955c6d4

packages/apps/kubernetes/images/kubevirt-csi-driver/Dockerfile

@@ -0,0 +1,25 @@
+# Source: https://github.com/kubevirt/csi-driver/blob/main/Dockerfile
+ARG builder_image=docker.io/library/golang:1.22.5
+FROM ${builder_image} AS builder
+RUN git clone https://github.com/kubevirt/csi-driver /src/kubevirt-csi-driver \
+ && cd /src/kubevirt-csi-driver \
+ && git checkout fa92820448e583c7fd722dc20270544e0c3eca53
+
+WORKDIR /src/kubevirt-csi-driver
+RUN make build
+
+FROM quay.io/centos/centos:stream9
+ARG git_url=https://github.com/kubevirt/csi-driver.git
+
+LABEL maintainers="The KubeVirt Project <kubevirt-dev@googlegroups.com>" \
+      description="KubeVirt CSI Driver" \
+      multi.GIT_URL=${git_url}
+
+ENTRYPOINT ["./kubevirt-csi-driver"]
+
+RUN dnf install -y e2fsprogs xfsprogs && dnf clean all
+
+ARG git_sha=NONE
+LABEL multi.GIT_SHA=${git_sha}
+
+COPY --from=builder /src/kubevirt-csi-driver/kubevirt-csi-driver .

packages/apps/kubernetes/images/ubuntu-container-disk.tag

@@ -1 +1 @@
-ghcr.io/aenix-io/cozystack/ubuntu-container-disk:v1.30.1@sha256:81caf89efe252ae2ca1990d08a3a314552d70ff36bcd4022b173c7150fbec805
+ghcr.io/aenix-io/cozystack/ubuntu-container-disk:v1.30.1@sha256:1f249fbe52821a62f706c6038b13401234e1b758ac498e53395b8f9a642b015f

packages/apps/kubernetes/templates/cloud-config.yaml

@@ -7,4 +7,6 @@ data:
     loadBalancer:
       creationPollInterval: 5
       creationPollTimeout: 60
+      enableEPSController: true
+      selectorless: true
     namespace: {{ .Release.Namespace }}

packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml

@@ -23,7 +23,7 @@ spec:
           operator: Exists
           effect: "NoSchedule"
       containers:
-      - image: ghcr.io/kvaps/test:cluster-autoscaller
+      - image: "{{ $.Files.Get "images/cluster-autoscaler.tag" | trim }}"
         name: cluster-autoscaler
         command:
         - /cluster-autoscaler

packages/apps/kubernetes/templates/cluster.yaml

@@ -18,6 +18,8 @@ spec:
       runStrategy: Always
       template:
         metadata:
+          annotations:
+            kubevirt.io/allow-pod-bridge-network-live-migration: "true"
           labels:
             {{- range .group.roles }}
             node-role.kubernetes.io/{{ . }}: ""
@@ -38,7 +40,9 @@ spec:
                 disk:
                   bus: virtio
                   pciAddress: 0000:08:00.0
-              networkInterfaceMultiqueue: true
+              interfaces:
+              - name: default
+                bridge: {}
             memory:
               guest: {{ .group.resources.memory }}
           evictionStrategy: External
@@ -49,6 +53,9 @@ spec:
           - name: ephemeral
             emptyDisk:
               capacity: {{ .group.ephemeralStorage | default "20Gi" }}
+          networks:
+          - name: default
+            pod: {}
 {{- end }}
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -203,6 +210,26 @@ spec:
         name: {{ $.Release.Name }}-{{ $groupName }}-{{ $kubevirtmachinetemplateHash }}
         namespace: default
       version: v1.30.1
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineHealthCheck
+metadata:
+  name: {{ $.Release.Name }}-{{ $groupName }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  clusterName: {{ $.Release.Name }}
+  nodeStartupTimeout: 10m
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: {{ $.Release.Name }}
+      cluster.x-k8s.io/deployment-name: {{ $.Release.Name }}-{{ $groupName }}
+  unhealthyConditions:
+  - type: Ready
+    status: Unknown
+    timeout: 30s
+  - type: Ready
+    status: "False"
+    timeout: 30s
 {{- end }}
 ---
 {{- /*

packages/apps/kubernetes/templates/csi/deploy.yaml

@@ -26,7 +26,7 @@ spec:
       containers:
         - name: csi-driver
           imagePullPolicy: Always
-          image: ghcr.io/kvaps/test:kubevirt-csi-driver
+          image: "{{ $.Files.Get "images/kubevirt-csi-driver.tag" | trim }}"
           args:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--infra-cluster-namespace=$(INFRACLUSTER_NAMESPACE)"
@@ -48,7 +48,7 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
             - name: INFRACLUSTER_LABELS
-              value: "csi-driver/cluster=test"
+              value: "cluster.x-k8s.io/cluster-name={{ .Release.Name }}"
             - name: INFRA_STORAGE_CLASS_ENFORCEMENT
               valueFrom:
                 configMapKeyRef:

packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml

@@ -29,6 +29,13 @@ spec:
   upgrade:
     remediation:
       retries: -1
+  {{- if .Values.addons.certManager.valuesOverride }}
+  valuesFrom:
+  - kind: Secret
+    name: {{ .Release.Name }}-cert-manager-values-override
+    valuesKey: values
+  {{- end }}
+
   dependsOn:
   {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
   - name: {{ .Release.Name }}
@@ -37,3 +44,13 @@ spec:
   - name: {{ .Release.Name }}-cilium
     namespace: {{ .Release.Namespace }}
 {{- end }}
+{{- if .Values.addons.certManager.valuesOverride }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-cert-manager-values-override
+stringData:
+  values: |
+    {{- toYaml .Values.addons.certManager.valuesOverride | nindent 4 }}
+{{- end }}

packages/apps/kubernetes/templates/helmreleases/cilium.yaml

@@ -30,21 +30,8 @@ spec:
       retries: -1
   values:
     cilium:
-      tunnel: disabled
-      autoDirectNodeRoutes: false
-      bpf:
-        masquerade: true
-      cgroup:
-        autoMount:
-          enabled: true
-        hostRoot: /run/cilium/cgroupv2
       k8sServiceHost: {{ .Release.Name }}.{{ .Release.Namespace }}.svc
       k8sServicePort: 6443
-    
-      cni:
-        chainingMode: ~
-        customConf: false
-        configMap: ""
       routingMode: tunnel
       enableIPv4Masquerade: true
       ipv4NativeRoutingCIDR: ""

packages/apps/kubernetes/templates/helmreleases/csi.yaml

@@ -28,6 +28,10 @@ spec:
   upgrade:
     remediation:
       retries: -1
+  {{- with .Values.storageClass }}
+  values:
+    storageClass: "{{ . }}"
+  {{- end }}
   dependsOn:
   {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
   - name: {{ .Release.Name }}

packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml

@@ -72,6 +72,12 @@ spec:
   upgrade:
     remediation:
       retries: -1
+  {{- if .Values.addons.fluxcd.valuesOverride }}
+  valuesFrom:
+  - kind: Secret
+    name: {{ .Release.Name }}-fluxcd-values-override
+    valuesKey: values
+  {{- end }}
   dependsOn:
   {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
   - name: {{ .Release.Name }}
@@ -82,3 +88,14 @@ spec:
   - name: {{ .Release.Name }}-fluxcd-operator
     namespace: {{ .Release.Namespace }}
 {{- end }}
+
+{{- if .Values.addons.fluxcd.valuesOverride }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-fluxcd-values-override
+stringData:
+  values: |
+    {{- toYaml .Values.addons.fluxcd.valuesOverride | nindent 4 }}
+{{- end }}

packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml

@@ -39,6 +39,12 @@ spec:
           enabled: false
       nodeSelector:
         node-role.kubernetes.io/ingress-nginx: ""
+  {{- if .Values.addons.ingressNginx.valuesOverride }}
+  valuesFrom:
+  - kind: Secret
+    name: {{ .Release.Name }}-ingress-nginx-values-override
+    valuesKey: values
+  {{- end }}
   dependsOn:
   {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
   - name: {{ .Release.Name }}
@@ -47,3 +53,14 @@ spec:
   - name: {{ .Release.Name }}-cilium
     namespace: {{ .Release.Namespace }}
 {{- end }}
+
+{{- if .Values.addons.ingressNginx.valuesOverride }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-ingress-nginx-values-override
+stringData:
+  values: |
+    {{- toYaml .Values.addons.ingressNginx.valuesOverride | nindent 4 }}
+{{- end }}

packages/apps/kubernetes/templates/kccm/kccm_role.yaml

@@ -34,6 +34,12 @@ rules:
   - services
   verbs:
   - "*"
+- apiGroups:
+  - "discovery.k8s.io"
+  resources:
+  - "endpointslices"
+  verbs:
+  - "*"
 - apiGroups:
   - ""
   resources:

packages/apps/kubernetes/templates/kccm/manager.yaml

@@ -30,7 +30,7 @@ spec:
         - --cluster-name={{ .Release.Name }}
         command:
         - /bin/kubevirt-cloud-controller-manager
-        image: ghcr.io/kvaps/test:kubevirt-cloud-provider
+        image: "{{ $.Files.Get "images/kubevirt-cloud-provider.tag" | trim }}"
         imagePullPolicy: Always
         #securityContext:
         #  privileged: true

packages/apps/kubernetes/values.schema.json

@@ -17,6 +17,11 @@
                 }
             }
         },
+        "storageClass": {
+            "type": "string",
+            "description": "StorageClass used to store user data",
+            "default": "replicated"
+        },
         "addons": {
             "type": "object",
             "properties": {
@@ -27,6 +32,11 @@
                             "type": "boolean",
                             "description": "Enables the cert-manager",
                             "default": false
+                        },
+                        "valuesOverride": {
+                            "type": "object",
+                            "description": "Custom values to override",
+                            "default": {}
                         }
                     }
                 },
@@ -38,6 +48,11 @@
                             "description": "Enable Ingress-NGINX controller (expect nodes with 'ingress-nginx' role)",
                             "default": false
                         },
+                        "valuesOverride": {
+                            "type": "object",
+                            "description": "Custom values to override",
+                            "default": {}
+                        },
                         "hosts": {
                             "type": "array",
                             "description": "List of domain names that should be passed through to the cluster by upper cluster",
@@ -53,6 +68,11 @@
                             "type": "boolean",
                             "description": "Enables Flux CD",
                             "default": false
+                        },
+                        "valuesOverride": {
+                            "type": "object",
+                            "description": "Custom values to override",
+                            "default": {}
                         }
                     }
                 }

packages/apps/kubernetes/values.yaml

@@ -2,10 +2,12 @@
 
 ## @param host The hostname used to access the Kubernetes cluster externally (defaults to using the cluster name as a subdomain for the tenant host).
 ## @param controlPlane.replicas Number of replicas for Kubernetes contorl-plane components
+## @param storageClass StorageClass used to store user data
 ##
 host: ""
 controlPlane:
   replicas: 2
+storageClass: replicated
 
 ## @param nodeGroups [object] nodeGroups configuration
 ##
@@ -28,12 +30,15 @@ addons:
   ##
   certManager:
     ## @param addons.certManager.enabled Enables the cert-manager
+    ## @param addons.certManager.valuesOverride Custom values to override
     enabled: false
+    valuesOverride: {}
 
   ## Ingress-NGINX Controller
   ##
   ingressNginx:
     ## @param addons.ingressNginx.enabled Enable Ingress-NGINX controller (expect nodes with 'ingress-nginx' role)
+    ## @param addons.ingressNginx.valuesOverride Custom values to override
     ##
     enabled: false
     ## @param addons.ingressNginx.hosts List of domain names that should be passed through to the cluster by upper cluster
@@ -43,10 +48,13 @@ addons:
     ## - foo.example.net
     ##
     hosts: []
+    valuesOverride: {}
 
   ## Flux CD
   ##
   fluxcd:
     ## @param addons.fluxcd.enabled Enables Flux CD
+    ## @param addons.fluxcd.valuesOverride Custom values to override
     ##
     enabled: false
+    valuesOverride: {}

packages/apps/mysql/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.0
+version: 0.5.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/mysql/Makefile

@@ -1,2 +1,20 @@
+MARIADB_BACKUP_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
+
+include ../../../scripts/common-envs.mk
+include ../../../scripts/package.mk
+
 generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md
+
+image:
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/mariadb-backup \
+		--provenance false \
+		--tag $(REGISTRY)/mariadb-backup:$(call settag,$(MARIADB_BACKUP_TAG)) \
+		--cache-from type=registry,ref=$(REGISTRY)/mariadb-backup:latest \
+		--cache-to type=inline \
+		--metadata-file images/mariadb-backup.json \
+		--push=$(PUSH) \
+		--load=$(LOAD)
+	echo "$(REGISTRY)/mariadb-backup:$(call settag,$(MARIADB_BACKUP_TAG))@$$(yq e '."containerimage.digest"' images/mariadb-backup.json -o json -r)" \
+		> images/mariadb-backup.tag
+	rm -f images/mariadb-backup.json

packages/apps/mysql/README.md

@@ -67,18 +67,19 @@ more details:
 
 ### Common parameters
 
-| Name       | Description                                     | Value   |
-| ---------- | ----------------------------------------------- | ------- |
-| `external` | Enable external access from outside the cluster | `false` |
-| `size`     | Persistent Volume size                          | `10Gi`  |
-| `replicas` | Number of MariaDB replicas                      | `2`     |
+| Name           | Description                                     | Value   |
+| -------------- | ----------------------------------------------- | ------- |
+| `external`     | Enable external access from outside the cluster | `false` |
+| `size`         | Persistent Volume size                          | `10Gi`  |
+| `replicas`     | Number of MariaDB replicas                      | `2`     |
+| `storageClass` | StorageClass used to store the data             | `""`    |
 
 ### Configuration parameters
 
 | Name        | Description             | Value |
 | ----------- | ----------------------- | ----- |
 | `users`     | Users configuration     | `{}`  |
-| `databases` | Databases configuration | `[]`  |
+| `databases` | Databases configuration | `{}`  |
 
 ### Backup parameters
 

packages/apps/mysql/images/mariadb-backup.tag

@@ -0,0 +1 @@
+ghcr.io/aenix-io/cozystack/mariadb-backup:0.5.1@sha256:793edb25a29cbc00781e40af883815ca36937e736e2b0d202ea9c9619fb6ca11

packages/apps/mysql/images/mariadb-backup/Dockerfile

@@ -0,0 +1,2 @@
+FROM alpine:3.20
+RUN apk add --no-cache mariadb-client uuidgen restic

packages/apps/mysql/templates/backup-cronjob.yaml

@@ -27,7 +27,7 @@ spec:
           restartPolicy: Never
           containers:
           - name: mysqldump
-            image: "{{ index $image "image.name" }}@{{ index $image "containerimage.digest" }}"
+            image: "{{ $.Files.Get "images/mariadb-backup.tag" | trim }}"
             command:
             - /bin/sh
             - /scripts/backup.sh

packages/apps/mysql/templates/dashboard-resourcemap.yaml

@@ -0,0 +1,20 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}-dashboard-resources
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  resourceNames:
+  - {{ .Release.Name }}-primary
+  - {{ .Release.Name }}-secondary
+  verbs: ["get", "list", "watch"]
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  resourceNames:
+  - {{ .Release.Name }}-credentials
+  verbs: ["get", "list", "watch"]

packages/apps/mysql/templates/db.yaml

@@ -1,14 +1,47 @@
-{{- range $name := .Values.databases }}
-{{ $dnsName := replace "_" "-" $name }}
+{{- range $name, $db := .Values.databases }}
+{{ $dbDNSName := replace "_" "-" $name }}
 ---
 apiVersion: k8s.mariadb.com/v1alpha1
 kind: Database
 metadata:
-  name: {{ $.Release.Name }}-{{ $dnsName }}
+  name: {{ $.Release.Name }}-{{ $dbDNSName }}
 spec:
   name: {{ $name }}
   mariaDbRef:
     name: {{ $.Release.Name }}
   characterSet: utf8
   collate: utf8_general_ci
+{{- range $user := $db.roles.admin }}
+{{ $userDNSName := replace "_" "-" $user }}
+---
+apiVersion: k8s.mariadb.com/v1alpha1
+kind: Grant
+metadata:
+  name: {{ $.Release.Name }}-{{ $dbDNSName }}-{{ $userDNSName }}
+spec:
+  mariaDbRef:
+    name: {{ $.Release.Name }}
+  privileges: ['ALL']
+  database: {{ $name }}
+  table: "*"
+  username: {{ $user }}
+  grantOption: true
+{{- end }}
+{{- range $user := $db.roles.readonly }}
+{{ $userDNSName := replace "_" "-" $user }}
+---
+apiVersion: k8s.mariadb.com/v1alpha1
+kind: Grant
+metadata:
+  name: {{ $.Release.Name }}-{{ $dbDNSName }}-{{ $userDNSName }}
+spec:
+  mariaDbRef:
+    name: {{ $.Release.Name }}
+  privileges: ['SELECT']
+  database: {{ $name }}
+  table: "*"
+  username: {{ $user }}
+  grantOption: true
+{{- end }}
+
 {{- end }}

packages/apps/mysql/templates/mariadb.yaml

@@ -4,11 +4,9 @@ kind: MariaDB
 metadata:
   name: {{ .Release.Name }}
 spec:
-  {{- if (and .Values.users.root .Values.users.root.password) }}
   rootPasswordSecretKeyRef:
-    name: {{ .Release.Name }}
-    key: root-password
-  {{- end }}
+    name: {{ .Release.Name }}-credentials
+    key: root
 
   image: "mariadb:11.0.2"
 
@@ -62,6 +60,9 @@ spec:
     size: {{ .Values.size }}
     resizeInUseVolumes: true
     waitForVolumeResize: true
+    {{- with .Values.storageClass }}
+    storageClassName: {{ . }}
+    {{- end }}
 
   {{- if .Values.external }}
   primaryService:

packages/apps/mysql/templates/secret.yaml

@@ -1,9 +1,31 @@
+{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (printf "%s-credentials" .Release.Name) }}
+{{- $passwords := dict }}
+
+{{- with (index $existingSecret "data") }}
+  {{- range $k, $v := . }}
+    {{- $_ := set $passwords $k (b64dec $v) }}
+  {{- end }}
+{{- end }}
+
+{{- $usersWithRoot := .Values.users }}
+{{- if (and .Values.users.root .Values.users.root.password) }}
+  {{- $_ := set $usersWithRoot "root" dict }}
+{{- end }}
+
+{{- range $user, $u := $usersWithRoot }}
+  {{- if $u.password }}
+    {{- $_ := set $passwords $user $u.password }}
+  {{- else if not (index $passwords $user) }}
+    {{- $_ := set $passwords $user (randAlphaNum 16) }}
+  {{- end }}
+{{- end }}
+
 ---
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ .Release.Name }}
+  name: {{ .Release.Name }}-credentials
 stringData:
-  {{- range $name, $u := .Values.users }}
-  {{ $name }}-password: {{ $u.password }}
+  {{- range $name, $u := $usersWithRoot }}
+  {{ $name }}: {{ index $passwords $name }}
   {{- end }}

packages/apps/mysql/templates/user.yaml

@@ -11,21 +11,8 @@ spec:
   mariaDbRef:
     name: {{ $.Release.Name }}
   passwordSecretKeyRef:
-    name: {{ $.Release.Name }}
-    key: {{ $name }}-password
+    name: {{ $.Release.Name }}-credentials
+    key: {{ $name }}
   maxUserConnections: {{ $u.maxUserConnections }}
----
-apiVersion: k8s.mariadb.com/v1alpha1
-kind: Grant
-metadata:
-  name: {{ $.Release.Name }}-{{ $dnsName }}
-spec:
-  mariaDbRef:
-    name: {{ $.Release.Name }}
-  privileges: {{ $u.privileges | toJson }}
-  database: "*"
-  table: "*"
-  username: {{ $name }}
-  grantOption: true
 {{- end }}
 {{- end }}

packages/apps/mysql/values.schema.json

@@ -17,11 +17,10 @@
             "description": "Number of MariaDB replicas",
             "default": 2
         },
-        "databases": {
-            "type": "array",
-            "description": "Databases configuration",
-            "default": [],
-            "items": {}
+        "storageClass": {
+            "type": "string",
+            "description": "StorageClass used to store the data",
+            "default": ""
         },
         "backup": {
             "type": "object",

packages/apps/mysql/values.yaml

@@ -3,37 +3,37 @@
 ## @param external Enable external access from outside the cluster
 ## @param size Persistent Volume size
 ## @param replicas Number of MariaDB replicas
+## @param storageClass StorageClass used to store the data
 ##
 external: false
 size: 10Gi
 replicas: 2
+storageClass: ""
 
 ## @section Configuration parameters
 
 ## @param users [object] Users configuration
 ## Example:
 ## users:
-##   root:
-##     password: strongpassword
 ##   user1:
-##     privileges: ['ALL']
 ##     maxUserConnections: 1000
 ##     password: hackme
 ##   user2:
-##     privileges: ['SELECT']
 ##     maxUserConnections: 1000
 ##     password: hackme
 ##
 users: {}
 
-## @param databases Databases configuration
+## @param databases [object] Databases configuration
 ## Example:
 ## databases:
-## - wordpress1
-## - wordpress2
-## - wordpress3
-## - wordpress4
-databases: []
+##   myapp1:
+##     roles:
+##       admin:
+##       - user1
+##       readonly:
+##       - user2
+databases: {}
 
 ## @section Backup parameters
 

packages/apps/nats/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to