[kubernetes] Pre-install NVIDIA GPU Driver

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

.github/CODEOWNERS

@@ -1 +1 @@
-* @kvaps @lllamnyp
+* @kvaps @lllamnyp @klinch0

.github/workflows/pre-commit.yml

@@ -1,13 +1,21 @@
 name: Pre-Commit Checks
 
-on: [push, pull_request]
+on:
-
+  push:
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - '**.md'
 jobs:
   pre-commit:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          fetch-tags: true
 
       - name: Set up Python
         uses: actions/setup-python@v4

.github/workflows/pull-requests-release.yaml

@@ -0,0 +1,96 @@
+name: Releasing PR
+
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened, closed]
+
+jobs:
+  verify:
+    name: Test Release
+    runs-on: [self-hosted]
+    permissions:
+      contents: read
+      packages: write
+
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'ok-to-test') &&
+      contains(github.event.pull_request.labels.*.name, 'release') &&
+      github.event.action != 'closed'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+
+      - name: Run tests
+        run: make test
+
+  finalize:
+    name: Finalize Release
+    runs-on: [self-hosted]
+    permissions:
+      contents: write
+  
+    if: |
+      github.event.pull_request.merged == true &&
+      contains(github.event.pull_request.labels.*.name, 'release')
+  
+    steps:
+      - name: Extract tag from branch name
+        id: get_tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const branch = context.payload.pull_request.head.ref;
+            const match = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
+
+            if (!match) {
+              core.setFailed(`Branch '${branch}' does not match expected format 'release-X.Y.Z[-suffix]'`);
+            } else {
+              const tag = `v${match[1]}`;
+              core.setOutput('tag', tag);
+              console.log(`✅ Extracted tag: ${tag}`);
+            }
+  
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+  
+      - name: Create tag on merged commit
+        run: |
+          git tag ${{ steps.get_tag.outputs.tag }} ${{ github.sha }} --force
+          git push origin ${{ steps.get_tag.outputs.tag }} --force
+  
+      - name: Publish draft release
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag = '${{ steps.get_tag.outputs.tag }}';
+            const releases = await github.rest.repos.listReleases({
+              owner: context.repo.owner,
+              repo: context.repo.repo
+            });
+  
+            const release = releases.data.find(r => r.tag_name === tag && r.draft);
+            if (!release) {
+              throw new Error(`Draft release with tag ${tag} not found`);
+            }
+  
+            await github.rest.repos.updateRelease({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              release_id: release.id,
+              draft: false
+            });
+  
+            console.log(`✅ Published release for ${tag}`);

.github/workflows/pull-requests.yaml

@@ -0,0 +1,39 @@
+name: Pull Request
+
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+
+jobs:
+  e2e:
+    name: Build and Test
+    runs-on: [self-hosted]
+    permissions:
+      contents: read
+      packages: write
+
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'ok-to-test') &&
+      !contains(github.event.pull_request.labels.*.name, 'release')
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+
+      - name: make build
+        run: |
+          make build
+
+      - name: make test
+        run: |
+          make test

.github/workflows/tags.yaml

@@ -0,0 +1,195 @@
+name: Versioned Tag
+
+on:
+  # Trigger on push if it includes a tag like vX.Y.Z
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  prepare-release:
+    name: Prepare Release
+    runs-on: [self-hosted]
+    permissions:
+      contents: write
+      packages: write
+      pull-requests: write
+
+    steps:
+      # 1) Check if a non-draft release with this tag already exists
+      - name: Check if release already exists
+        id: check_release
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag = context.ref.replace('refs/tags/', '');
+            const releases = await github.rest.repos.listReleases({
+              owner: context.repo.owner,
+              repo: context.repo.repo
+            });
+            const existing = releases.data.find(r => r.tag_name === tag && !r.draft);
+            if (existing) {
+              core.setOutput('skip', 'true');
+            } else {
+              core.setOutput('skip', 'false');
+            }
+
+      # If a published release already exists, skip the rest of the workflow
+      - name: Skip if release already exists
+        if: steps.check_release.outputs.skip == 'true'
+        run: echo "Release already exists, skipping workflow."
+
+      # 2) Determine the base branch from which the tag was pushed
+      - name: Get base branch
+        if: steps.check_release.outputs.skip == 'false'
+        id: get_base
+        uses: actions/github-script@v7
+        with:
+          script: |
+            /*
+              For a push event with a tag, GitHub sets context.payload.base_ref 
+              if the tag was pushed from a branch.
+              If it's empty, we can't determine the correct base branch and must fail.
+            */
+            const baseRef = context.payload.base_ref;
+            if (!baseRef) {
+              core.setFailed(`❌ base_ref is empty. Make sure you push the tag from a branch (e.g. 'git push origin HEAD:refs/tags/vX.Y.Z').`);
+              return;
+            }
+
+            const shortBranch = baseRef.replace("refs/heads/", "");
+            const releasePattern = /^release-\d+\.\d+$/;
+            if (shortBranch !== "main" && !releasePattern.test(shortBranch)) {
+              core.setFailed(`❌ Tagged commit must belong to branch 'main' or 'release-X.Y'. Got '${shortBranch}'`);
+              return;
+            }
+
+            core.setOutput('branch', shortBranch);
+
+      # 3) Checkout full git history and tags
+      - name: Checkout code
+        if: steps.check_release.outputs.skip == 'false'
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      # 4) Login to GitHub Container Registry
+      - name: Login to GitHub Container Registry
+        if: steps.check_release.outputs.skip == 'false'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+
+      # 5) Build project artifacts
+      - name: Build
+        if: steps.check_release.outputs.skip == 'false'
+        run: make build
+
+      # 6) Optionally commit built artifacts to the repository
+      - name: Commit release artifacts
+        if: steps.check_release.outputs.skip == 'false'
+        env:
+          GIT_AUTHOR_NAME: ${{ github.actor }}
+          GIT_AUTHOR_EMAIL: ${{ github.actor }}@users.noreply.github.com
+        run: |
+          git config user.name "github-actions"
+          git config user.email "github-actions@github.com"
+          git add .
+          git commit -m "Prepare release ${GITHUB_REF#refs/tags/}" -s || echo "No changes to commit"
+
+      # 7) Create a release branch like release-X.Y.Z
+      - name: Create release branch
+        if: steps.check_release.outputs.skip == 'false'
+        run: |
+          BRANCH_NAME="release-${GITHUB_REF#refs/tags/v}"
+          git branch -f "$BRANCH_NAME"
+          git push origin "$BRANCH_NAME" --force
+
+      # 8) Create a pull request from release-X.Y.Z to the original base branch
+      - name: Create pull request if not exists
+        if: steps.check_release.outputs.skip == 'false'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const version = context.ref.replace('refs/tags/v', '');
+            const base = '${{ steps.get_base.outputs.branch }}';
+            const head = `release-${version}`;
+
+            const prs = await github.rest.pulls.list({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              head: `${context.repo.owner}:${head}`,
+              base
+            });
+
+            if (prs.data.length === 0) {
+              const newPr = await github.rest.pulls.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                head,
+                base,
+                title: `Release v${version}`,
+                body:
+                  `This PR prepares the release \`v${version}\`.\n` +
+                  `(Please merge it before releasing draft)`,
+                draft: false
+              });
+
+              console.log(`Created pull request #${newPr.data.number} from ${head} to ${base}`);
+              await github.rest.issues.addLabels({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: newPr.data.number,
+                labels: ['release']
+              });
+            } else {
+              console.log(`Pull request already exists from ${head} to ${base}`);
+            }
+
+      # 9) Create or reuse an existing draft GitHub release for this tag
+      - name: Create or reuse draft release
+        if: steps.check_release.outputs.skip == 'false'
+        id: create_release
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag = context.ref.replace('refs/tags/', '');
+            const releases = await github.rest.repos.listReleases({
+              owner: context.repo.owner,
+              repo: context.repo.repo
+            });
+
+            let release = releases.data.find(r => r.tag_name === tag);
+            if (!release) {
+              release = await github.rest.repos.createRelease({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                tag_name: tag,
+                name: `${tag}`,
+                draft: true,
+                prerelease: false
+              });
+            }
+            core.setOutput('upload_url', release.upload_url);
+
+      # 10) Build additional assets for the release (if needed)
+      - name: Build assets
+        if: steps.check_release.outputs.skip == 'false'
+        run: make assets
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # 11) Upload assets to the draft release
+      - name: Upload assets
+        if: steps.check_release.outputs.skip == 'false'
+        run: make upload_assets VERSION=${GITHUB_REF#refs/tags/}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # 12) Run tests
+      - name: Run tests
+        if: steps.check_release.outputs.skip == 'false'
+        run: make test

CONTRIBUTING.md

@@ -6,13 +6,13 @@ As you get started, you are in the best position to give us feedbacks on areas o
 
 * Problems found while setting up the development environment
 * Gaps in our documentation
-* Bugs in our Github actions
+* Bugs in our GitHub actions
 
-First, though, it is important that you read the [code of conduct](CODE_OF_CONDUCT.md).
+First, though, it is important that you read the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
 
 The guidelines below are a starting point. We don't want to limit your
 creativity, passion, and initiative. If you think there's a better way, please
-feel free to bring it up in a Github discussion, or open a pull request. We're
+feel free to bring it up in a GitHub discussion, or open a pull request. We're
 certain there are always better ways to do things, we just need to start some
 constructive dialogue!
 
@@ -23,9 +23,9 @@ We welcome many types of contributions including:
 * New features
 * Builds, CI/CD
 * Bug fixes
-* [Documentation](https://github.com/cozystack/cozystack-website/tree/main)
+* [Documentation](https://GitHub.com/cozystack/cozystack-website/tree/main)
 * Issue Triage
-* Answering questions on Slack or Github Discussions
+* Answering questions on Slack or GitHub Discussions
 * Web design
 * Communications / Social Media / Blog Posts
 * Events participation
@@ -34,7 +34,7 @@ We welcome many types of contributions including:
 ## Ask for Help
 
 The best way to reach us with a question when contributing is to drop a line in
-our [Telegram channel](https://t.me/cozystack), or start a new Github discussion.
+our [Telegram channel](https://t.me/cozystack), or start a new GitHub discussion.
 
 ## Raising Issues
 

Makefile

@@ -1,6 +1,13 @@
 .PHONY: manifests repos assets
 
-build:
+build-deps:
+	@command -V find docker skopeo jq gh helm > /dev/null
+	@yq --version | grep -q "mikefarah" || (echo "mikefarah/yq is required" && exit 1)
+	@tar --version | grep -q GNU || (echo "GNU tar is required" && exit 1)
+	@sed --version | grep -q GNU || (echo "GNU sed is required" && exit 1)
+	@awk --version | grep -q GNU || (echo "GNU awk is required" && exit 1)
+
+build: build-deps
 	make -C packages/apps/http-cache image
 	make -C packages/apps/postgres image
 	make -C packages/apps/mysql image
@@ -19,10 +26,6 @@ build:
 	make -C packages/core/installer image
 	make manifests
 
-manifests:
-	mkdir -p _out/assets
-	(cd packages/core/installer/; helm template -n cozy-installer installer .) > _out/assets/cozystack-installer.yaml
-
 repos:
 	rm -rf _out
 	make -C packages/apps check-version-map
@@ -33,17 +36,21 @@ repos:
 	mkdir -p _out/logos
 	cp ./packages/apps/*/logos/*.svg ./packages/extra/*/logos/*.svg _out/logos/
 
+
+manifests:
+	mkdir -p _out/assets
+	(cd packages/core/installer/; helm template -n cozy-installer installer .) > _out/assets/cozystack-installer.yaml
+
 assets:
 	make -C packages/core/installer/ assets
 
 test:
-	test -f _out/assets/nocloud-amd64.raw.xz || make -C packages/core/installer talos-nocloud
 	make -C packages/core/testing apply
 	make -C packages/core/testing test
-	make -C packages/core/testing test-applications
+	#make -C packages/core/testing test-applications
 
 generate:
 	hack/update-codegen.sh
 
-upload_assets: assets
+upload_assets: manifests
 	hack/upload-assets.sh

README.md

@@ -12,20 +12,21 @@
 
 **Cozystack** is a free PaaS platform and framework for building clouds.
 
-With Cozystack, you can transform your bunch of servers into an intelligent system with a simple REST API for spawning Kubernetes clusters, Database-as-a-Service, virtual machines, load balancers, HTTP caching services, and other services with ease.
+With Cozystack, you can transform a bunch of servers into an intelligent system with a simple REST API for spawning Kubernetes clusters,
+Database-as-a-Service, virtual machines, load balancers, HTTP caching services, and other services with ease.
 
-You can use Cozystack to build your own cloud or to provide a cost-effective development environments.  
+Use Cozystack to build your own cloud or provide a cost-effective development environment.  
 
 ## Use-Cases
 
-* [**Using Cozystack to build public cloud**](https://cozystack.io/docs/use-cases/public-cloud/)  
+* [**Using Cozystack to build a public cloud**](https://cozystack.io/docs/guides/use-cases/public-cloud/)  
-You can use Cozystack as backend for a public cloud
+You can use Cozystack as a backend for a public cloud
 
-* [**Using Cozystack to build private cloud**](https://cozystack.io/docs/use-cases/private-cloud/)  
+* [**Using Cozystack to build a private cloud**](https://cozystack.io/docs/guides/use-cases/private-cloud/)  
-You can use Cozystack as platform to build a private cloud powered by Infrastructure-as-Code approach
+You can use Cozystack as a platform to build a private cloud powered by Infrastructure-as-Code approach
 
-* [**Using Cozystack as Kubernetes distribution**](https://cozystack.io/docs/use-cases/kubernetes-distribution/)  
+* [**Using Cozystack as a Kubernetes distribution**](https://cozystack.io/docs/guides/use-cases/kubernetes-distribution/)  
-You can use Cozystack as Kubernetes distribution for Bare Metal
+You can use Cozystack as a Kubernetes distribution for Bare Metal
 
 ## Screenshot
 
@@ -33,11 +34,11 @@ You can use Cozystack as Kubernetes distribution for Bare Metal
 
 ## Documentation
 
-The documentation is located on official [cozystack.io](https://cozystack.io) website.
+The documentation is located on the [cozystack.io](https://cozystack.io) website.
 
-Read [Get Started](https://cozystack.io/docs/get-started/) section for a quick start.
+Read the [Getting Started](https://cozystack.io/docs/getting-started/) section for a quick start.
 
-If you encounter any difficulties, start with the [troubleshooting guide](https://cozystack.io/docs/troubleshooting/), and work your way through the process that we've outlined.
+If you encounter any difficulties, start with the [troubleshooting guide](https://cozystack.io/docs/operations/troubleshooting/) and work your way through the process that we've outlined.
 
 ## Versioning
 
@@ -50,15 +51,15 @@ A full list of the available releases is available in the GitHub repository's [R
 
 Contributions are highly appreciated and very welcomed!
 
-In case of bugs, please, check if the issue has been already opened by checking the [GitHub Issues](https://github.com/cozystack/cozystack/issues) section.
+In case of bugs, please check if the issue has already been opened by checking the [GitHub Issues](https://github.com/cozystack/cozystack/issues) section.
-In case it isn't, you can open a new one: a detailed report will help us to replicate it, assess it, and work on a fix.
+If it isn't, you can open a new one. A detailed report will help us replicate it, assess it, and work on a fix.
 
-You can express your intention in working on the fix on your own.
+You can express your intention to on the fix on your own.
 Commits are used to generate the changelog, and their author will be referenced in it.
 
-In case of **Feature Requests** please use the [Discussion's Feature Request section](https://github.com/cozystack/cozystack/discussions/categories/feature-requests).
+If you have **Feature Requests** please use the [Discussion's Feature Request section](https://github.com/cozystack/cozystack/discussions/categories/feature-requests).
 
-You can join our weekly community meetings (just add this events to your [Google Calendar](https://calendar.google.com/calendar?cid=ZTQzZDIxZTVjOWI0NWE5NWYyOGM1ZDY0OWMyY2IxZTFmNDMzZTJlNjUzYjU2ZGJiZGE3NGNhMzA2ZjBkMGY2OEBncm91cC5jYWxlbmRhci5nb29nbGUuY29t) or [iCal](https://calendar.google.com/calendar/ical/e43d21e5c9b45a95f28c5d649c2cb1e1f433e2e653b56dbbda74ca306f0d0f68%40group.calendar.google.com/public/basic.ics)) or [Telegram group](https://t.me/cozystack).
+You are welcome to join our weekly community meetings (just add this events to your [Google Calendar](https://calendar.google.com/calendar?cid=ZTQzZDIxZTVjOWI0NWE5NWYyOGM1ZDY0OWMyY2IxZTFmNDMzZTJlNjUzYjU2ZGJiZGE3NGNhMzA2ZjBkMGY2OEBncm91cC5jYWxlbmRhci5nb29nbGUuY29t) or [iCal](https://calendar.google.com/calendar/ical/e43d21e5c9b45a95f28c5d649c2cb1e1f433e2e653b56dbbda74ca306f0d0f68%40group.calendar.google.com/public/basic.ics)) or [Telegram group](https://t.me/cozystack).
 
 ## License
 

cmd/cozystack-controller/main.go

@@ -178,6 +178,15 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "WorkloadMonitor")
 		os.Exit(1)
 	}
+
+	if err = (&controller.WorkloadReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Workload")
+		os.Exit(1)
+	}
+
 	// +kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

dashboards/flux/flux-control-plane.json

@@ -626,7 +626,7 @@
             "datasource": {
               "uid": "${DS_PROMETHEUS}"
             },
-            "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"POD\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)",
+            "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)",
             "hide": false,
             "interval": "",
             "legendFormat": "{{pod}}",

dashboards/main/capacity-planning.json

@@ -450,7 +450,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval])))\n / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))",
+          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"\",node=~\"$node\"}[$__rate_interval])))\n / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -520,7 +520,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"})) / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",unit=\"byte\",node=~\"$node\"}[$__rate_interval])))",
+          "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"\",node=~\"$node\"})) / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",unit=\"byte\",node=~\"$node\"}[$__rate_interval])))",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -590,7 +590,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval]))) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))",
+          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"\",node=~\"$node\"}[$__rate_interval]))) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -660,7 +660,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"} )) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"}[$__rate_interval])))",
+          "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"\",node=~\"$node\"} )) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"}[$__rate_interval])))",
           "hide": false,
           "legendFormat": "__auto",
           "range": true,
@@ -1128,7 +1128,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0\n",
+          "expr": "sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0\n",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -1143,7 +1143,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0)",
+          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0)",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -1527,7 +1527,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0\n",
+          "expr": "(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0\n",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -1542,7 +1542,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum((sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0)",
+          "expr": "sum((sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0)",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -1909,7 +1909,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) * -1 > 0)\n",
+          "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) * -1 > 0)\n",
           "format": "table",
           "instant": true,
           "range": false,
@@ -2037,7 +2037,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) * -1 >0)\n",
+          "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) * -1 >0)\n",
           "format": "table",
           "instant": true,
           "range": false,
@@ -2160,7 +2160,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) > 0)\n",
+          "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) > 0)\n",
           "format": "table",
           "instant": true,
           "range": false,
@@ -2288,7 +2288,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) >0)\n",
+          "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) >0)\n",
           "format": "table",
           "instant": true,
           "range": false,

dashboards/main/controller.json

@@ -684,7 +684,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])  * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])  * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -710,7 +710,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (pod)\n(\n  avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\", namespace=\"$namespace\", resource=\"cpu\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "sum by (pod)\n(\n  avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\", namespace=\"$namespace\", resource=\"cpu\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -723,7 +723,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n    -\n    sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n  ) > 0\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n    -\n    sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n  ) > 0\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -736,7 +736,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) \n  (\n    (\n      (\n        sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n      ) or sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n    ) > 0\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) \n  (\n    (\n      (\n        sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n      ) or sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n    ) > 0\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -762,7 +762,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -786,7 +786,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (pod)\n(\n  avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\", namespace=\"$namespace\", resource=\"memory\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "sum by (pod)\n(\n  avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\", namespace=\"$namespace\", resource=\"memory\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -798,7 +798,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    (\n      (\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n      ) > 0\n    )\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    (\n      (\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n      ) > 0\n    )\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -810,7 +810,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    (\n      (\n        sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n      ) or sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n    ) > 0\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    (\n      (\n        sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n      ) or sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n    ) > 0\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -848,7 +848,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -860,7 +860,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -1315,7 +1315,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -1488,7 +1488,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -1502,7 +1502,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -1642,7 +1642,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by (pod)\n  (\n    max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n    * on (pod)\n    sum by (pod) (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (pod)\n  (\n    max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n    * on (pod)\n    sum by (pod) (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -1779,7 +1779,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "    (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (\n    (\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n    )\n    or\n    sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n  )\n) > 0",
+          "expr": "    (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (\n    (\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n    )\n    or\n    sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n  )\n) > 0",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -2095,7 +2095,7 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Usage",
@@ -2109,7 +2109,7 @@
               "refId": "D"
             },
             {
-              "expr": "sum by (pod)\n(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\", namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))\n)",
+              "expr": "sum by (pod)\n(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\", namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))\n)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
@@ -2295,7 +2295,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by(pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "System",
@@ -2306,7 +2306,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by(pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "User",
@@ -2468,7 +2468,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -2653,7 +2653,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "RSS",
@@ -2666,7 +2666,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Cache",
@@ -2679,7 +2679,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Swap",
@@ -2692,7 +2692,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Working set bytes without kmem",
@@ -2705,7 +2705,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Kmem",
@@ -2837,7 +2837,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (pod) group_left()\n  sum by (pod)\n    (\n      (\n        sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n        -\n        sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n      ) > 0\n    )\n)",
+          "expr": "(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (pod) group_left()\n  sum by (pod)\n    (\n      (\n        sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n        -\n        sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n      ) > 0\n    )\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -2974,7 +2974,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (pod) group_left()\n  sum by (pod)\n    (\n      (\n        (\n          sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n        ) or sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n      ) > 0\n    )\n)",
+          "expr": "(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (pod) group_left()\n  sum by (pod)\n    (\n      (\n        (\n          sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n        ) or sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n      ) > 0\n    )\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -3290,56 +3290,56 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "RSS",
               "refId": "A"
             },
             {
-              "expr": "sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Cache",
               "refId": "B"
             },
             {
-              "expr": "sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Swap",
               "refId": "C"
             },
             {
-              "expr": "sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Working set bytes without kmem",
               "refId": "D"
             },
             {
-              "expr": "sum by (pod)\n(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", container!=\"POD\", resource=\"memory\"}[$__rate_interval]))\n)",
+              "expr": "sum by (pod)\n(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", container!=\"\", resource=\"memory\"}[$__rate_interval]))\n)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
               "refId": "E"
             },
             {
-              "expr": "sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "F"
             },
             {
-              "expr": "sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "G"
             },
             {
-              "expr": "sum by(pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Kmem",
@@ -3834,7 +3834,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -3972,7 +3972,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",

dashboards/main/namespace.json

@@ -656,7 +656,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -680,7 +680,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"cpu\"}[$__range]))\n  ) \nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"cpu\"}[$__range]))\n  ) \nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -692,7 +692,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n          -\n          sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n          -\n          sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -704,7 +704,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n          ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n          ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -728,7 +728,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -740,7 +740,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (pod) group_left()\n    sum by (namespace, pod)\n      (\n        avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__range])\n      )\n    )\n  or\n  count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (pod) group_left()\n    sum by (namespace, pod)\n      (\n        avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__range])\n      )\n    )\n  or\n  count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -752,7 +752,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"memory\"}[$__range]))\n  ) \n  or  \ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"memory\"}[$__range]))\n  ) \n  or  \ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -764,7 +764,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n          -\n          sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n          -\n          sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -776,7 +776,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n          ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n          ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -814,7 +814,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -826,7 +826,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -877,7 +877,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -1475,7 +1475,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval])))",
+          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval])))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -1646,7 +1646,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum (sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))))",
+          "expr": "sum (sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "System",
@@ -1657,7 +1657,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum (sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))))",
+          "expr": "sum (sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "User",
@@ -1798,7 +1798,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -1939,7 +1939,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n          ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n          ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -2257,28 +2257,28 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval])))",
+              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval])))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Usage",
               "refId": "D"
             },
             {
-              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
+              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "C"
             },
             {
-              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
+              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "E"
             },
             {
-              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))\n  )",
+              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
@@ -2458,7 +2458,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval])))",
+              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval])))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -2470,7 +2470,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval])))",
+              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval])))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "User",
@@ -2622,7 +2622,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -2799,14 +2799,14 @@
       "pluginVersion": "8.5.13",
       "targets": [
         {
-          "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "RSS",
           "refId": "A"
         },
         {
-          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -2814,7 +2814,7 @@
           "refId": "B"
         },
         {
-          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -2822,14 +2822,14 @@
           "refId": "C"
         },
         {
-          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Working set bytes without kmem",
           "refId": "D"
         },
         {
-          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Kmem",
@@ -2955,7 +2955,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -3091,7 +3091,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n          )\n          or\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n            +\n            sum by(namespace, pod, container) (avg_over_time(container_memory:kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n          )\n        ) > 0\n      )\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n          )\n          or\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n            +\n            sum by(namespace, pod, container) (avg_over_time(container_memory:kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n          )\n        ) > 0\n      )\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -3408,14 +3408,14 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "RSS",
               "refId": "A"
             },
             {
-              "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} \n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} \n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3423,7 +3423,7 @@
               "refId": "B"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3431,35 +3431,35 @@
               "refId": "C"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Working set bytes without kmem",
               "refId": "D"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\",namespace=\"$namespace\"}[$__rate_interval]))\n  ) ",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))\n  ) ",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "E"
             },
             {
-              "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} \n    * on (pod) group_left() \n    sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
+              "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} \n    * on (pod) group_left() \n    sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "F"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"memory\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"memory\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
               "refId": "G"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Kmem",
@@ -3910,7 +3910,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])))",
+          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval])))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -4049,7 +4049,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])))",
+          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval])))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",

dashboards/main/namespaces.json

@@ -869,7 +869,7 @@
           "refId": "A"
         },
         {
-          "expr": "100 * count by (namespace) (\n  sum by (namespace, verticalpodautoscaler) (  \n    count by (namespace, controller_name, verticalpodautoscaler) (avg_over_time(vpa_target_recommendation{namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\n    / on (controller_name, namespace) group_left\n    count by (namespace, controller_name) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range]))\n  )\n) \n/ count by (namespace) (sum by (namespace, controller) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "100 * count by (namespace) (\n  sum by (namespace, verticalpodautoscaler) (  \n    count by (namespace, controller_name, verticalpodautoscaler) (avg_over_time(vpa_target_recommendation{namespace=~\"$namespace\", container!=\"\"}[$__range]))\n    / on (controller_name, namespace) group_left\n    count by (namespace, controller_name) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range]))\n  )\n) \n/ count by (namespace) (sum by (namespace, controller) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -878,7 +878,7 @@
           "refId": "B"
         },
         {
-          "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -895,7 +895,7 @@
           "refId": "D"
         },
         {
-          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n    ) > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n    ) > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -903,7 +903,7 @@
           "refId": "E"
         },
         {
-          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n    )\n    > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n    )\n    > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -919,7 +919,7 @@
           "refId": "G"
         },
         {
-          "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -935,7 +935,7 @@
           "refId": "I"
         },
         {
-          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n    ) > 0\n  )\nor\ncount(avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n    ) > 0\n  )\nor\ncount(avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -943,7 +943,7 @@
           "refId": "J"
         },
         {
-          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n    )\n    > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n    )\n    > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -968,7 +968,7 @@
           "refId": "M"
         },
         {
-          "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -977,7 +977,7 @@
           "refId": "N"
         },
         {
-          "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -1449,7 +1449,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -1616,7 +1616,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "System",
@@ -1627,7 +1627,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "User",
@@ -1764,7 +1764,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -1901,7 +1901,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n    )\n    > 0\n  )",
+          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n    )\n    > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -2210,7 +2210,7 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -2218,21 +2218,21 @@
               "refId": "A"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})",
+              "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "B"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})",
+              "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "C"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(vpa_target_recommendation{container!=\"POD\", namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(vpa_target_recommendation{container!=\"\", namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
@@ -2407,7 +2407,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by (namespace) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -2419,7 +2419,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by (namespace) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "User",
@@ -2572,7 +2572,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -2754,14 +2754,14 @@
       "pluginVersion": "8.5.13",
       "targets": [
         {
-          "expr": "sum (avg_over_time(container_memory_rss{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory_rss{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "RSS",
           "refId": "A"
         },
         {
-          "expr": "sum (avg_over_time(container_memory_cache{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory_cache{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -2769,7 +2769,7 @@
           "refId": "B"
         },
         {
-          "expr": "sum (avg_over_time(container_memory_swap{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory_swap{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -2777,14 +2777,14 @@
           "refId": "C"
         },
         {
-          "expr": "sum (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Working set bytes without kmem",
           "refId": "D"
         },
         {
-          "expr": "sum (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Kmem",
@@ -2910,7 +2910,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -3046,7 +3046,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n    )\n    > 0\n  )",
+          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n    )\n    > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -3370,14 +3370,14 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "RSS",
               "refId": "A"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3385,7 +3385,7 @@
               "refId": "B"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3393,35 +3393,35 @@
               "refId": "C"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Working set bytes without kmem",
               "refId": "D"
             },
             {
-              "expr": "sum by(namespace) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"memory\"}[$__rate_interval]))",
+              "expr": "sum by(namespace) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"memory\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
               "refId": "E"
             },
             {
-              "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "F"
             },
             {
-              "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "G"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Kmem",
@@ -3873,7 +3873,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -4008,7 +4008,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",

dashboards/main/pod.json

@@ -686,7 +686,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
+          "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -759,7 +759,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
+          "expr": "sum by (container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -847,7 +847,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by(container) (rate(container_fs_reads_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__range]))",
+          "expr": "sum by(container) (rate(container_fs_reads_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__range]))",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -860,7 +860,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by(container) (rate(container_fs_writes_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__range]))",
+          "expr": "sum by(container) (rate(container_fs_writes_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__range]))",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -899,7 +899,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
+          "expr": "sum by (container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -1503,7 +1503,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -1669,7 +1669,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(pod) (rate(container_cpu_system_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (rate(container_cpu_system_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -1681,7 +1681,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(pod) (rate(container_cpu_user_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (rate(container_cpu_user_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "User",
@@ -1820,7 +1820,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace, pod, container)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{container!=\"POD\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (namespace, pod, container)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{container!=\"\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -2269,7 +2269,7 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Usage",
@@ -2476,7 +2476,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_cpu_system_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_cpu_system_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -2488,7 +2488,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_cpu_user_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_cpu_user_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "User",
@@ -2639,7 +2639,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -2816,7 +2816,7 @@
       "pluginVersion": "8.5.13",
       "targets": [
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -2824,28 +2824,28 @@
           "refId": "A"
         },
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Cache",
           "refId": "B"
         },
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Swap",
           "refId": "C"
         },
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Working set bytes without kmem",
           "refId": "D"
         },
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Kmem",
@@ -2974,7 +2974,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (container)\n  (\n    (\n      sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (container)\n  (\n    (\n      sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -3110,7 +3110,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (container)\n  (\n    (\n      (\n        sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n        -\n        sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))\n      ) or sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (container)\n  (\n    (\n      (\n        sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n        -\n        sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"\"}[$__rate_interval]))\n      ) or sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -3431,7 +3431,7 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by(container) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "instant": false,
               "intervalFactor": 1,
@@ -3439,7 +3439,7 @@
               "refId": "A"
             },
             {
-              "expr": "sum by(container) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3447,28 +3447,28 @@
               "refId": "B"
             },
             {
-              "expr": "sum by(container) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Swap",
               "refId": "C"
             },
             {
-              "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Working set bytes without kmem",
               "refId": "D"
             },
             {
-              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "E"
             },
             {
-              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
@@ -3482,7 +3482,7 @@
               "refId": "G"
             },
             {
-              "expr": "sum by(container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Kmem",
@@ -3930,7 +3930,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ container }}",
@@ -4068,7 +4068,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_fs_writes_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_fs_writes_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ container }}",

docs/release.md

@@ -0,0 +1,139 @@
+# Release Workflow
+
+This section explains how Cozystack builds and releases are made.
+
+## Regular Releases
+
+When making regular releases, we take a commit in `main` and decide to make it a release `x.y.0`.
+In this explanation, we'll use version `v0.42.0` as an example:
+
+```mermaid
+gitGraph
+    commit id: "feature"
+    commit id: "feature 2"
+    commit id: "feature 3" tag: "v0.42.0"
+```
+
+A regular release sequence starts in the following way:
+
+1. Maintainer tags a commit in `main` with `v0.42.0` and pushes it to GitHub.
+2. CI workflow triggers on tag push:
+   1. Creates a draft page for release `v0.42.0`, if it wasn't created before.
+   2. Takes code from tag `v0.42.0`, builds images, and pushes them to ghcr.io.
+   3. Makes a new commit `Prepare release v0.42.0` with updated digests, pushes it to the new branch `release-0.42.0`, and opens a PR to `main`.
+   4. Builds Cozystack release assets from the new commit `Prepare release v0.42.0` and uploads them to the release draft page.
+3. Maintainer reviews PR, tests build artifacts, and edits changelogs on the release draft page.
+
+   ```mermaid
+   gitGraph
+       commit id: "feature"
+       commit id: "feature 2"
+       commit id: "feature 3" tag: "v0.42.0"
+       branch release-0.42.0
+       checkout release-0.42.0
+       commit id: "Prepare release v0.42.0"
+       checkout main
+       merge release-0.42.0 id: "Pull Request"
+   ```
+
+   When testing and editing are completed, the sequence goes on.
+
+4. Maintainer merges the PR. GitHub removes the merged branch `release-0.42.0`.
+5. CI workflow triggers on merge:
+   1. Moves the tag `v0.42.0` to the newly created merge commit by force-pushing a tag to GitHub.
+   2. Publishes the release page (`draft` → `latest`).
+6. The maintainer can now announce the release to the community.
+
+```mermaid
+gitGraph
+    commit id: "feature"
+    commit id: "feature 2"
+    commit id: "feature 3"
+    branch release-0.42.0
+    checkout release-0.42.0
+    commit id: "Prepare release v0.42.0"
+    checkout main
+    merge release-0.42.0 id: "Release v0.42.0" tag: "v0.42.0"
+```
+
+## Patch Releases
+
+Making a patch release has a lot in common with a regular release, with a couple of differences:
+
+* A release branch is used instead of `main`
+* Patch commits are cherry-picked to the release branch.
+* A pull request is opened against the release branch.
+
+
+Let's assume that we've released `v0.42.0` and that development is ongoing.
+We have introduced a couple of new features and some fixes to features that we have released 
+in `v0.42.0`.
+
+Once problems were found and fixed, a patch release is due.
+
+```mermaid
+gitGraph
+   commit id: "Release v0.42.0" tag: "v0.42.0"
+    checkout main
+    commit id: "feature 4"
+    commit id: "patch 1"
+    commit id: "feature 5"
+    commit id: "patch 2"
+```
+
+
+1. The maintainer creates a release branch, `release-0.42,` and cherry-picks patch commits from `main` to `release-0.42`.
+   These must be only patches to features that were present in version `v0.42.0`.
+
+   Cherry-picking can be done as soon as each patch is merged into `main`,
+   or directly before the release.
+
+   ```mermaid
+   gitGraph
+       commit id: "Release v0.42.0" tag: "v0.42.0"
+       branch release-0.42
+       checkout main
+       commit id: "feature 4"
+       commit id: "patch 1"
+       commit id: "feature 5"
+       commit id: "patch 2"
+       checkout release-0.42
+       cherry-pick id: "patch 1"
+       cherry-pick id: "patch 2"
+   ```
+
+   When all relevant patch commits are cherry-picked, the branch is ready for release.
+
+2. The maintainer tags the `HEAD` commit of branch `release-0.42` as `v0.42.1` and then pushes it to GitHub.
+3. CI workflow triggers on tag push:
+    1. Creates a draft page for release `v0.42.1`, if it wasn't created before.
+    2. Takes code from tag `v0.42.1`, builds images, and pushes them to ghcr.io.
+    3. Makes a new commit `Prepare release v0.42.1` with updated digests, pushes it to the new branch `release-0.42.1`, and opens a PR to `release-0.42`.
+    4. Builds Cozystack release assets from the new commit `Prepare release v0.42.1` and uploads them to the release draft page.
+4. Maintainer reviews PR, tests build artifacts, and edits changelogs on the release draft page.
+   
+   ```mermaid
+   gitGraph
+       commit id: "Release v0.42.0" tag: "v0.42.0"
+       branch release-0.42
+       checkout main
+       commit id: "feature 4"
+       commit id: "patch 1"
+       commit id: "feature 5"
+       commit id: "patch 2"
+       checkout release-0.42
+       cherry-pick id: "patch 1"
+       cherry-pick id: "patch 2" tag: "v0.42.1"
+       branch release-0.42.1
+       commit id: "Prepare release v0.42.1"
+       checkout release-0.42
+       merge release-0.42.1 id: "Pull request"
+   ```
+
+   Finally, when release is confirmed, the release sequence goes on.
+
+5. Maintainer merges the PR. GitHub removes the merged branch `release-0.42.1`.
+6. CI workflow triggers on merge:
+   1. Moves the tag `v0.42.1` to the newly created merge commit by force-pushing a tag to GitHub.
+   2. Publishes the release page (`draft` → `latest`).
+7. The maintainer can now announce the release to the community.

hack/e2e.sh

@@ -84,7 +84,7 @@ done
 
 # Start VMs
 for i in 1 2 3; do
-  qemu-system-x86_64 -machine type=pc,accel=kvm -cpu host -smp 4 -m 8192 \
+  qemu-system-x86_64 -machine type=pc,accel=kvm -cpu host -smp 8 -m 16384 \
     -device virtio-net,netdev=net0,mac=52:54:00:12:34:5$i -netdev tap,id=net0,ifname=cozy-srv$i,script=no,downscript=no \
     -drive file=srv$i/system.img,if=virtio,format=raw \
     -drive file=srv$i/seed.img,if=virtio,format=raw \
@@ -113,6 +113,11 @@ machine:
         - usermode_helper=disabled
     - name: zfs
     - name: spl
+  registries:
+    mirrors:
+      docker.io:
+        endpoints:
+        - https://mirror.gcr.io
   files:
   - content: |
       [plugins]
@@ -313,7 +318,12 @@ kubectl patch -n tenant-root tenants.apps.cozystack.io root --type=merge -p '{"s
 timeout 60 sh -c 'until kubectl get hr -n tenant-root etcd ingress monitoring tenant-root; do sleep 1; done'
 
 # Wait for HelmReleases be installed
-kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr etcd ingress monitoring tenant-root
+kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr etcd ingress tenant-root
+
+if ! kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr monitoring; then
+  flux reconcile hr monitoring -n tenant-root --force
+  kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr monitoring
+fi
 
 kubectl patch -n tenant-root ingresses.apps.cozystack.io ingress --type=merge -p '{"spec":{
   "dashboard": true
@@ -328,7 +338,7 @@ kubectl wait --timeout=5m --for=jsonpath=.status.readyReplicas=3 -n tenant-root
 
 # Wait for Victoria metrics
 kubectl wait --timeout=5m --for=jsonpath=.status.updateStatus=operational -n tenant-root vmalert/vmalert-shortterm vmalertmanager/alertmanager
-kubectl wait --timeout=5m --for=jsonpath=.status.status=operational -n tenant-root vlogs/generic
+kubectl wait --timeout=5m --for=jsonpath=.status.updateStatus=operational -n tenant-root vlogs/generic
 kubectl wait --timeout=5m --for=jsonpath=.status.clusterStatus=operational -n tenant-root vmcluster/shortterm vmcluster/longterm
 
 # Wait for grafana

hack/gen_versions_map.sh

@@ -19,21 +19,19 @@ fi
 miss_map=$(echo "$new_map" | awk 'NR==FNR { nm[$1 " " $2] = $3; next } { if (!($1 " " $2 in nm)) print $1, $2, $3}' - "$file")
 
 # search accross all tags sorted by version
-search_commits=$(git ls-remote --tags origin | grep 'refs/tags/v' | sort -k2,2 -rV | awk '{print $1}')
+search_commits=$(git ls-remote --tags origin | awk -F/ '$3 ~ /v[0-9]+.[0-9]+.[0-9]+/ {print}' | sort -k2,2 -rV | awk '{print $1}')
-# add latest main commit to search
-search_commits="${search_commits} $(git rev-parse "origin/main")"
 
 resolved_miss_map=$(
   echo "$miss_map" | while read -r chart version commit; do
     # if version is found in HEAD, it's HEAD
-    if grep -q "^version: $version$" ./${chart}/Chart.yaml; then
+    if [ "$(awk '$1 == "version:" {print $2}' ./${chart}/Chart.yaml)" = "${version}" ]; then
       echo "$chart $version HEAD"
       continue
     fi
 
     # if commit is not HEAD, check if it's valid
-    if [ $commit != "HEAD" ]; then
+    if [ "$commit" != "HEAD" ]; then
-      if ! git show "${commit}:./${chart}/Chart.yaml" 2>/dev/null | grep -q "^version: $version$"; then
+      if [ "$(git show "${commit}:./${chart}/Chart.yaml" | awk '$1 == "version:" {print $2}')" != "${version}" ]; then
         echo "Commit $commit for $chart $version is not valid" >&2
         exit 1
       fi
@@ -46,15 +44,15 @@ resolved_miss_map=$(
     # if commit is HEAD, but version is not found in HEAD, check all tags
     found_tag=""
     for tag in $search_commits; do
-      if git show "${tag}:./${chart}/Chart.yaml" 2>/dev/null | grep -q "^version: $version$"; then
+      if [ "$(git show "${tag}:./${chart}/Chart.yaml" | awk '$1 == "version:" {print $2}')" = "${version}" ]; then
         found_tag=$(git rev-parse --short "${tag}")
         break
       fi
     done
     
     if [ -z "$found_tag" ]; then
-      echo "Can't find $chart $version in any version tag or in the latest main commit" >&2
+      echo "Can't find $chart $version in any version tag, removing it" >&2
-      exit 1
+      continue
     fi
     
     echo "$chart $version $found_tag"

hack/upload-assets.sh

@@ -1,8 +1,11 @@
 #!/bin/bash
 set -xe
 
-version=$(git describe --tags)
+version=${VERSION:-$(git describe --tags)}
-gh release upload $version _out/assets/cozystack-installer.yaml
+
-gh release upload $version _out/assets/metal-amd64.iso
+gh release upload --clobber $version _out/assets/cozystack-installer.yaml
-gh release upload $version _out/assets/metal-amd64.raw.xz
+gh release upload --clobber $version _out/assets/metal-amd64.iso
-gh release upload $version _out/assets/nocloud-amd64.raw.xz
+gh release upload --clobber $version _out/assets/metal-amd64.raw.xz
+gh release upload --clobber $version _out/assets/nocloud-amd64.raw.xz
+gh release upload --clobber $version _out/assets/kernel-amd64
+gh release upload --clobber $version _out/assets/initramfs-metal-amd64.xz

internal/controller/workload_controller.go

@@ -0,0 +1,87 @@
+package controller
+
+import (
+	"context"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
+)
+
+// WorkloadMonitorReconciler reconciles a WorkloadMonitor object
+type WorkloadReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+}
+
+func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+	w := &cozyv1alpha1.Workload{}
+	err := r.Get(ctx, req.NamespacedName, w)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return ctrl.Result{}, nil
+		}
+		logger.Error(err, "Unable to fetch Workload")
+		return ctrl.Result{}, err
+	}
+
+	// it's being deleted, nothing to handle
+	if w.DeletionTimestamp != nil {
+		return ctrl.Result{}, nil
+	}
+
+	t := getMonitoredObject(w)
+	err = r.Get(ctx, types.NamespacedName{Name: t.GetName(), Namespace: t.GetNamespace()}, t)
+
+	// found object, nothing to do
+	if err == nil {
+		return ctrl.Result{}, nil
+	}
+
+	// error getting object but not 404 -- requeue
+	if !apierrors.IsNotFound(err) {
+		logger.Error(err, "failed to get dependent object", "kind", t.GetObjectKind(), "dependent-object-name", t.GetName())
+		return ctrl.Result{}, err
+	}
+
+	err = r.Delete(ctx, w)
+	if err != nil {
+		logger.Error(err, "failed to delete workload")
+	}
+	return ctrl.Result{}, err
+}
+
+// SetupWithManager registers our controller with the Manager and sets up watches.
+func (r *WorkloadReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		// Watch WorkloadMonitor objects
+		For(&cozyv1alpha1.Workload{}).
+		Complete(r)
+}
+
+func getMonitoredObject(w *cozyv1alpha1.Workload) client.Object {
+	if strings.HasPrefix(w.Name, "pvc-") {
+		obj := &corev1.PersistentVolumeClaim{}
+		obj.Name = strings.TrimPrefix(w.Name, "pvc-")
+		obj.Namespace = w.Namespace
+		return obj
+	}
+	if strings.HasPrefix(w.Name, "svc-") {
+		obj := &corev1.Service{}
+		obj.Name = strings.TrimPrefix(w.Name, "svc-")
+		obj.Namespace = w.Namespace
+		return obj
+	}
+	obj := &corev1.Pod{}
+	obj.Name = w.Name
+	obj.Namespace = w.Namespace
+	return obj
+}

internal/controller/workloadmonitor_controller.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"sort"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -33,6 +34,17 @@ type WorkloadMonitorReconciler struct {
 // +kubebuilder:rbac:groups=cozystack.io,resources=workloads,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=cozystack.io,resources=workloads/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch
+
+// isServiceReady checks if the service has an external IP bound
+func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool {
+	return len(svc.Status.LoadBalancer.Ingress) > 0
+}
+
+// isPVCReady checks if the PVC is bound
+func (r *WorkloadMonitorReconciler) isPVCReady(pvc *corev1.PersistentVolumeClaim) bool {
+	return pvc.Status.Phase == corev1.ClaimBound
+}
 
 // isPodReady checks if the Pod is in the Ready condition.
 func (r *WorkloadMonitorReconciler) isPodReady(pod *corev1.Pod) bool {
@@ -88,6 +100,110 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) {
 	obj.SetOwnerReferences(owners)
 }
 
+// reconcileServiceForMonitor creates or updates a Workload object for the given Service and WorkloadMonitor.
+func (r *WorkloadMonitorReconciler) reconcileServiceForMonitor(
+	ctx context.Context,
+	monitor *cozyv1alpha1.WorkloadMonitor,
+	svc corev1.Service,
+) error {
+	logger := log.FromContext(ctx)
+	workload := &cozyv1alpha1.Workload{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("svc-%s", svc.Name),
+			Namespace: svc.Namespace,
+		},
+	}
+
+	resources := make(map[string]resource.Quantity)
+
+	quantity := resource.MustParse("0")
+
+	for _, ing := range svc.Status.LoadBalancer.Ingress {
+		if ing.IP != "" {
+			quantity.Add(resource.MustParse("1"))
+		}
+	}
+
+	var resourceLabel string
+	if svc.Annotations != nil {
+		var ok bool
+		resourceLabel, ok = svc.Annotations["metallb.universe.tf/ip-allocated-from-pool"]
+		if !ok {
+			resourceLabel = "default"
+		}
+	}
+	resourceLabel = fmt.Sprintf("%s.ipaddresspool.metallb.io/requests.ipaddresses", resourceLabel)
+	resources[resourceLabel] = quantity
+
+	_, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error {
+		// Update owner references with the new monitor
+		updateOwnerReferences(workload.GetObjectMeta(), monitor)
+
+		workload.Labels = svc.Labels
+
+		// Fill Workload status fields:
+		workload.Status.Kind = monitor.Spec.Kind
+		workload.Status.Type = monitor.Spec.Type
+		workload.Status.Resources = resources
+		workload.Status.Operational = r.isServiceReady(&svc)
+
+		return nil
+	})
+	if err != nil {
+		logger.Error(err, "Failed to CreateOrUpdate Workload", "workload", workload.Name)
+		return err
+	}
+
+	return nil
+}
+
+// reconcilePVCForMonitor creates or updates a Workload object for the given PVC and WorkloadMonitor.
+func (r *WorkloadMonitorReconciler) reconcilePVCForMonitor(
+	ctx context.Context,
+	monitor *cozyv1alpha1.WorkloadMonitor,
+	pvc corev1.PersistentVolumeClaim,
+) error {
+	logger := log.FromContext(ctx)
+	workload := &cozyv1alpha1.Workload{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("pvc-%s", pvc.Name),
+			Namespace: pvc.Namespace,
+		},
+	}
+
+	resources := make(map[string]resource.Quantity)
+
+	for resourceName, resourceQuantity := range pvc.Status.Capacity {
+		storageClass := "default"
+		if pvc.Spec.StorageClassName != nil || *pvc.Spec.StorageClassName == "" {
+			storageClass = *pvc.Spec.StorageClassName
+		}
+		resourceLabel := fmt.Sprintf("%s.storageclass.storage.k8s.io/requests.%s", storageClass, resourceName.String())
+		resources[resourceLabel] = resourceQuantity
+	}
+
+	_, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error {
+		// Update owner references with the new monitor
+		updateOwnerReferences(workload.GetObjectMeta(), monitor)
+
+		workload.Labels = pvc.Labels
+
+		// Fill Workload status fields:
+		workload.Status.Kind = monitor.Spec.Kind
+		workload.Status.Type = monitor.Spec.Type
+		workload.Status.Resources = resources
+		workload.Status.Operational = r.isPVCReady(&pvc)
+
+		return nil
+	})
+	if err != nil {
+		logger.Error(err, "Failed to CreateOrUpdate Workload", "workload", workload.Name)
+		return err
+	}
+
+	return nil
+}
+
 // reconcilePodForMonitor creates or updates a Workload object for the given Pod and WorkloadMonitor.
 func (r *WorkloadMonitorReconciler) reconcilePodForMonitor(
 	ctx context.Context,
@@ -205,6 +321,45 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		}
 	}
 
+	pvcList := &corev1.PersistentVolumeClaimList{}
+	if err := r.List(
+		ctx,
+		pvcList,
+		client.InNamespace(monitor.Namespace),
+		client.MatchingLabels(monitor.Spec.Selector),
+	); err != nil {
+		logger.Error(err, "Unable to list PVCs for WorkloadMonitor", "monitor", monitor.Name)
+		return ctrl.Result{}, err
+	}
+
+	for _, pvc := range pvcList.Items {
+		if err := r.reconcilePVCForMonitor(ctx, monitor, pvc); err != nil {
+			logger.Error(err, "Failed to reconcile Workload for PVC", "PVC", pvc.Name)
+			continue
+		}
+	}
+
+	svcList := &corev1.ServiceList{}
+	if err := r.List(
+		ctx,
+		svcList,
+		client.InNamespace(monitor.Namespace),
+		client.MatchingLabels(monitor.Spec.Selector),
+	); err != nil {
+		logger.Error(err, "Unable to list Services for WorkloadMonitor", "monitor", monitor.Name)
+		return ctrl.Result{}, err
+	}
+
+	for _, svc := range svcList.Items {
+		if svc.Spec.Type != corev1.ServiceTypeLoadBalancer {
+			continue
+		}
+		if err := r.reconcileServiceForMonitor(ctx, monitor, svc); err != nil {
+			logger.Error(err, "Failed to reconcile Workload for Service", "Service", svc.Name)
+			continue
+		}
+	}
+
 	// Update WorkloadMonitor status based on observed pods
 	monitor.Status.ObservedReplicas = observedReplicas
 	monitor.Status.AvailableReplicas = availableReplicas
@@ -233,41 +388,51 @@ func (r *WorkloadMonitorReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		// Also watch Pod objects and map them back to WorkloadMonitor if labels match
 		Watches(
 			&corev1.Pod{},
-			handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
+			handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(&corev1.Pod{}, r.Client)),
-				pod, ok := obj.(*corev1.Pod)
+		).
-				if !ok {
+		// Watch PVCs as well
-					return nil
+		Watches(
-				}
+			&corev1.PersistentVolumeClaim{},
-
+			handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(&corev1.PersistentVolumeClaim{}, r.Client)),
-				var monitorList cozyv1alpha1.WorkloadMonitorList
-				// List all WorkloadMonitors in the same namespace
-				if err := r.List(ctx, &monitorList, client.InNamespace(pod.Namespace)); err != nil {
-					return nil
-				}
-
-				// Match each monitor's selector with the Pod's labels
-				var requests []reconcile.Request
-				for _, m := range monitorList.Items {
-					matches := true
-					for k, v := range m.Spec.Selector {
-						if podVal, exists := pod.Labels[k]; !exists || podVal != v {
-							matches = false
-							break
-						}
-					}
-					if matches {
-						requests = append(requests, reconcile.Request{
-							NamespacedName: types.NamespacedName{
-								Namespace: m.Namespace,
-								Name:      m.Name,
-							},
-						})
-					}
-				}
-				return requests
-			}),
 		).
 		// Watch for changes to Workload objects we create (owned by WorkloadMonitor)
 		Owns(&cozyv1alpha1.Workload{}).
 		Complete(r)
 }
+
+func mapObjectToMonitor[T client.Object](_ T, c client.Client) func(ctx context.Context, obj client.Object) []reconcile.Request {
+	return func(ctx context.Context, obj client.Object) []reconcile.Request {
+		concrete, ok := obj.(T)
+		if !ok {
+			return nil
+		}
+
+		var monitorList cozyv1alpha1.WorkloadMonitorList
+		// List all WorkloadMonitors in the same namespace
+		if err := c.List(ctx, &monitorList, client.InNamespace(concrete.GetNamespace())); err != nil {
+			return nil
+		}
+
+		labels := concrete.GetLabels()
+		// Match each monitor's selector with the Pod's labels
+		var requests []reconcile.Request
+		for _, m := range monitorList.Items {
+			matches := true
+			for k, v := range m.Spec.Selector {
+				if labelVal, exists := labels[k]; !exists || labelVal != v {
+					matches = false
+					break
+				}
+			}
+			if matches {
+				requests = append(requests, reconcile.Request{
+					NamespacedName: types.NamespacedName{
+						Namespace: m.Namespace,
+						Name:      m.Name,
+					},
+				})
+			}
+		}
+		return requests
+	}
+}

packages/apps/clickhouse/images/clickhouse-backup.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/clickhouse-backup:0.6.2@sha256:67dd53efa86b704fc5cb876aca055fef294b31ab67899b683a4821ea12582ea7
+ghcr.io/cozystack/cozystack/clickhouse-backup:0.7.0@sha256:3faf7a4cebf390b9053763107482de175aa0fdb88c1e77424fd81100b1c3a205

packages/apps/ferretdb/images/postgres-backup.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/postgres-backup:0.9.0@sha256:2b6ba87f5688a439bd2ac12835a5ab9e601feb15c0c44ed0d9ca48cec7c52521
+ghcr.io/cozystack/cozystack/postgres-backup:0.10.0@sha256:10179ed56457460d95cd5708db2a00130901255fa30c4dd76c65d2ef5622b61f

packages/apps/http-cache/images/nginx-cache.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/nginx-cache:0.3.1@sha256:2b82eae28239ca0f9968602c69bbb752cd2a5818e64934ccd06cb91d95d019c7
+ghcr.io/cozystack/cozystack/nginx-cache:0.4.0@sha256:bef7344da098c4dc400a9e20ffad10ac991df67d09a30026207454abbc91f28b

packages/apps/kubernetes/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.17.0
+version: 0.18.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/kubernetes/images/cluster-autoscaler.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/cluster-autoscaler:0.15.2@sha256:967e51702102d0dbd97f9847de4159d62681b31eb606322d2c29755393c2236e
+ghcr.io/cozystack/cozystack/cluster-autoscaler:0.18.0@sha256:85371c6aabf5a7fea2214556deac930c600e362f92673464fe2443784e2869c3

packages/apps/kubernetes/images/kubevirt-cloud-provider.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:latest@sha256:47ad85a2bb2b11818df85e80cbc6e07021e97e429d5bb020ce8db002b37a77f1
+ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:0.18.0@sha256:795d8e1ef4b2b0df2aa1e09d96cd13476ebb545b4bf4b5779b7547a70ef64cf9

packages/apps/kubernetes/images/kubevirt-csi-driver.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.15.2@sha256:cb4ab74099662f73e058f7c7495fb403488622c3425c06ad23b687bfa8bc805b
+ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.18.0@sha256:6f9091c3e7e4951c5e43fdafd505705fcc9f1ead290ee3ae42e97e9ec2b87b20

packages/apps/kubernetes/images/ubuntu-container-disk.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30.1@sha256:bc08ea0ced2cb7dd98b26d72a9462fc0a3863adb908a5effbfcdf7227656ea65
+ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30.1@sha256:07392e7a87a3d4ef1c86c1b146e6c5de5c2b524aed5a53bf48870dc8a296f99a

packages/apps/kubernetes/images/ubuntu-container-disk/Dockerfile

@@ -17,6 +17,9 @@ RUN wget -O image.img https://cloud-images.ubuntu.com/jammy/current/jammy-server
 
 RUN qemu-img resize image.img 5G \
  && eval "$(guestfish --listen --network)" \
+ && guestfish --remote set-backend direct \
+ && guestfish --remote set-memsize 2048 \
+ && guestfish --remote set-smp 4 \
  && guestfish --remote add-drive image.img \
  && guestfish --remote run \
  && guestfish --remote mount /dev/sda1 / \
@@ -37,6 +40,9 @@ RUN qemu-img resize image.img 5G \
  && guestfish --remote command "sed -i '/SystemdCgroup/ s/=.*/= true/' /etc/containerd/config.toml" \
 # install kubernetes
  && guestfish --remote command "apt-get install -y kubelet kubeadm" \
+# install nvidia driver
+ && guestfish --remote command "apt-get install -y nvidia-dkms-570-server-open" \
+ && guestfish --remote sh "printf 'blacklist %s\n' nvidia nvidia_uvm nvidia_drm nvidia_modeset > /etc/modprobe.d/blacklist-nvidia.conf" \
 # clean apt cache
  && guestfish --remote sh 'apt-get clean && rm -rf /var/lib/apt/lists/*' \
 # write system configuration

packages/apps/kubernetes/templates/helmreleases/delete.yaml

@@ -32,6 +32,9 @@ spec:
                 {{ .Release.Name }}-cilium
                 {{ .Release.Name }}-csi
                 {{ .Release.Name }}-cert-manager
+                {{ .Release.Name }}-cert-manager-crds
+                {{ .Release.Name }}-vertical-pod-autoscaler
+                {{ .Release.Name }}-vertical-pod-autoscaler-crds
                 {{ .Release.Name }}-ingress-nginx
                 {{ .Release.Name }}-fluxcd-operator
                 {{ .Release.Name }}-fluxcd
@@ -67,6 +70,9 @@ rules:
       - {{ .Release.Name }}-cilium
       - {{ .Release.Name }}-csi
       - {{ .Release.Name }}-cert-manager
+      - {{ .Release.Name }}-cert-manager-crds
+      - {{ .Release.Name }}-vertical-pod-autoscaler
+      - {{ .Release.Name }}-vertical-pod-autoscaler-crds
       - {{ .Release.Name }}-ingress-nginx
       - {{ .Release.Name }}-fluxcd-operator
       - {{ .Release.Name }}-fluxcd

packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml

@@ -38,10 +38,10 @@ spec:
   - name: {{ .Release.Name }}
     namespace: {{ .Release.Namespace }}
   {{- end }}
-  - name: {{ .Release.Name }}-cilium
-    namespace: {{ .Release.Namespace }}
   - name: {{ .Release.Name }}-cozy-victoria-metrics-operator
     namespace: {{ .Release.Namespace }}
+  - name: {{ .Release.Name }}-vertical-pod-autoscaler-crds
+    namespace: {{ .Release.Namespace }}
   values:
     vmagent:
       externalLabels:

packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml

@@ -0,0 +1,41 @@
+{{- if .Values.addons.monitoringAgents.enabled }}
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: {{ .Release.Name }}-vertical-pod-autoscaler-crds
+  labels:
+    cozystack.io/repository: system
+    coztstack.io/target-cluster-name: {{ .Release.Name }}
+spec:
+  interval: 5m
+  releaseName: vertical-pod-autoscaler-crds
+  chart:
+    spec:
+      chart: cozy-vertical-pod-autoscaler-crds
+      reconcileStrategy: Revision
+      sourceRef:
+        kind: HelmRepository
+        name: cozystack-system
+        namespace: cozy-system
+  kubeConfig:
+    secretRef:
+      name: {{ .Release.Name }}-admin-kubeconfig
+      key: super-admin.svc
+  targetNamespace: cozy-vertical-pod-autoscaler-crds
+  storageNamespace: cozy-vertical-pod-autoscaler-crds
+  install:
+    createNamespace: true
+    remediation:
+      retries: -1
+  upgrade:
+    remediation:
+      retries: -1
+
+  dependsOn:
+  {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
+  - name: {{ .Release.Name }}
+    namespace: {{ .Release.Namespace }}
+  {{- end }}
+  - name: {{ .Release.Name }}-cilium
+    namespace: {{ .Release.Namespace }}
+{{- end }}

packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml

@@ -0,0 +1,69 @@
+{{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }}
+{{- $targetTenant := index $myNS.metadata.annotations "namespace.cozystack.io/monitoring" }}
+{{- if .Values.addons.monitoringAgents.enabled }}
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: {{ .Release.Name }}-vertical-pod-autoscaler
+  labels:
+    cozystack.io/repository: system
+    coztstack.io/target-cluster-name: {{ .Release.Name }}
+spec:
+  interval: 5m
+  releaseName: vertical-pod-autoscaler
+  chart:
+    spec:
+      chart: cozy-vertical-pod-autoscaler
+      reconcileStrategy: Revision
+      sourceRef:
+        kind: HelmRepository
+        name: cozystack-system
+        namespace: cozy-system
+  kubeConfig:
+    secretRef:
+      name: {{ .Release.Name }}-admin-kubeconfig
+      key: super-admin.svc
+  targetNamespace: cozy-vertical-pod-autoscaler
+  storageNamespace: cozy-vertical-pod-autoscaler
+  install:
+    createNamespace: true
+    remediation:
+      retries: -1
+  upgrade:
+    remediation:
+      retries: -1
+  values:
+    vertical-pod-autoscaler:
+      recommender:
+        extraArgs:
+          container-name-label: container
+          container-namespace-label: namespace
+          container-pod-name-label: pod
+          storage: prometheus
+          memory-saver: true
+          pod-label-prefix: label_
+          metric-for-pod-labels: kube_pod_labels{job="kube-state-metrics", tenant="{{ .Release.Namespace }}", cluster="{{ .Release.Name }}"}[8d]
+          pod-name-label: pod
+          pod-namespace-label: namespace
+          prometheus-address: http://vmselect-shortterm.{{ $targetTenant }}.svc.cozy.local:8481/select/0/prometheus/
+          prometheus-cadvisor-job-name: cadvisor
+        resources:
+          limits:
+            memory: 1600Mi
+          requests:
+            cpu: 100m
+            memory: 1600Mi
+  {{- if .Values.addons.verticalPodAutoscaler.valuesOverride }}
+  valuesFrom:
+  - kind: Secret
+    name: {{ .Release.Name }}-vertical-pod-autoscaler-values-override
+    valuesKey: values
+  {{- end }}
+  dependsOn:
+  {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
+  - name: {{ .Release.Name }}
+    namespace: {{ .Release.Namespace }}
+  {{- end }}
+  - name: {{ .Release.Name }}-monitoring-agents
+    namespace: {{ .Release.Namespace }}
+{{- end }}

packages/apps/kubernetes/values.yaml

@@ -70,6 +70,13 @@ addons:
     enabled: false
     valuesOverride: {}
 
+  ## VerticalPodAutoscaler
+  ##
+  verticalPodAutoscaler:
+    ## @param addons.verticalPodAutoscaler.valuesOverride Custom values to override
+    ##
+    valuesOverride: {}
+
 ## @section Kamaji control plane
 ##
 kamajiControlPlane:
@@ -85,7 +92,7 @@ kamajiControlPlane:
     #     memory: 512Mi
     
     ## @param kamajiControlPlane.apiServer.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).
-    resourcesPreset: "micro"
+    resourcesPreset: "small"
 
   controllerManager:
     ## @param kamajiControlPlane.controllerManager.resources Resources

packages/apps/mysql/images/mariadb-backup.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/mariadb-backup:0.5.3@sha256:8ca1fb01e880d351ee7d984a0b437c1142836963cd079986156ed28750067138
+ghcr.io/cozystack/cozystack/mariadb-backup:0.6.0@sha256:cfd1c37d8ad24e10681d82d6e6ce8a641b4602c1b0ffa8516ae15b4958bb12d4

packages/apps/postgres/images/postgres-backup.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/postgres-backup:0.9.0@sha256:2b6ba87f5688a439bd2ac12835a5ab9e601feb15c0c44ed0d9ca48cec7c52521
+ghcr.io/cozystack/cozystack/postgres-backup:0.10.0@sha256:10179ed56457460d95cd5708db2a00130901255fa30c4dd76c65d2ef5622b61f

packages/apps/tenant/templates/configuration-hash.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cozy-tenant-configuration-hash
+  namespace: {{ include "tenant.name" . }}
+data:
+  cozyTenantConfigurationHash: {{ sha256sum (toJson .Values) | quote }}

packages/apps/versions_map

@@ -56,7 +56,10 @@ kubernetes 0.15.0 4e68e65c
 kubernetes 0.15.1 160e4e2a
 kubernetes 0.15.2 8267072d
 kubernetes 0.16.0 077045b0
-kubernetes 0.17.0 HEAD
+kubernetes 0.17.0 1fbbfcd0
+kubernetes 0.17.1 fd240701
+kubernetes 0.18.0 721c12a7
+kubernetes 0.18.1 HEAD
 mysql 0.1.0 263e47be
 mysql 0.2.0 c24a103f
 mysql 0.3.0 53f2365e
@@ -139,15 +142,18 @@ virtual-machine 0.7.0 e23286a3
 virtual-machine 0.7.1 0ab39f20
 virtual-machine 0.8.0 3fa4dd3a
 virtual-machine 0.8.1 93c46161
-virtual-machine 0.8.2 HEAD
+virtual-machine 0.8.2 de19450f
-vm-disk 0.1.0 HEAD
+virtual-machine 0.9.0 HEAD
+vm-disk 0.1.0 d971f2ff
+vm-disk 0.1.1 HEAD
 vm-instance 0.1.0 1ec10165
 vm-instance 0.2.0 84f3ccc0
 vm-instance 0.3.0 4e68e65c
 vm-instance 0.4.0 e23286a3
 vm-instance 0.4.1 0ab39f20
 vm-instance 0.5.0 3fa4dd3a
-vm-instance 0.5.1 HEAD
+vm-instance 0.5.1 de19450f
+vm-instance 0.6.0 HEAD
 vpn 0.1.0 263e47be
 vpn 0.2.0 53f2365e
 vpn 0.3.0 6c5cf5bf

packages/apps/virtual-machine/Chart.yaml

@@ -17,10 +17,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.8.2
+version: 0.9.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.8.2"
+appVersion: 0.9.0

packages/apps/virtual-machine/Makefile

@@ -2,6 +2,7 @@ include ../../../scripts/package.mk
 
 generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md
+	yq -o json -i '.properties.gpus.items.type = "object" | .properties.gpus.default = []' values.schema.json
 	INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \
 	  && yq -i -o json ".properties.instanceType.optional=true | .properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json
 	PREFERENCES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/preferences.yaml | yq 'split(" ") | . + [""]' -o json) \

packages/apps/virtual-machine/README.md

@@ -36,22 +36,23 @@ virtctl ssh <user>@<vm>
 
 ### Common parameters
 
-| Name                      | Description                                                                                                | Value            |
+| Name                      | Description                                                                                                | Value        |
-| ------------------------- | ---------------------------------------------------------------------------------------------------------- | ---------------- |
+| ------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------ |
-| `external`                | Enable external access from outside the cluster                                                            | `false`          |
+| `external`                | Enable external access from outside the cluster                                                            | `false`      |
-| `externalMethod`          | specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` | `WholeIP`        |
+| `externalMethod`          | specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` | `WholeIP`    |
-| `externalPorts`           | Specify ports to forward from outside the cluster                                                          | `[]`             |
+| `externalPorts`           | Specify ports to forward from outside the cluster                                                          | `[]`         |
-| `running`                 | Determines if the virtual machine should be running                                                        | `true`           |
+| `running`                 | Determines if the virtual machine should be running                                                        | `true`       |
-| `instanceType`            | Virtual Machine instance type                                                                              | `u1.medium`      |
+| `instanceType`            | Virtual Machine instance type                                                                              | `u1.medium`  |
-| `instanceProfile`         | Virtual Machine prefferences profile                                                                       | `ubuntu`         |
+| `instanceProfile`         | Virtual Machine preferences profile                                                                        | `ubuntu`     |
-| `systemDisk.image`        | The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos` | `ubuntu`         |
+| `systemDisk.image`        | The base image for the virtual machine. Allowed values: `ubuntu`, `cirros`, `alpine`, `fedora` and `talos` | `ubuntu`     |
-| `systemDisk.storage`      | The size of the disk allocated for the virtual machine                                                     | `5Gi`            |
+| `systemDisk.storage`      | The size of the disk allocated for the virtual machine                                                     | `5Gi`        |
-| `systemDisk.storageClass` | StorageClass used to store the data                                                                        | `replicated`     |
+| `systemDisk.storageClass` | StorageClass used to store the data                                                                        | `replicated` |
-| `resources.cpu`           | The number of CPU cores allocated to the virtual machine                                                   | `""`             |
+| `gpus`                    | List of GPUs to attach                                                                                     | `[]`         |
-| `resources.memory`        | The amount of memory allocated to the virtual machine                                                      | `""`             |
+| `resources.cpu`           | The number of CPU cores allocated to the virtual machine                                                   | `""`         |
-| `sshKeys`                 | List of SSH public keys for authentication. Can be a single key or a list of keys.                         | `[]`             |
+| `resources.memory`        | The amount of memory allocated to the virtual machine                                                      | `""`         |
-| `cloudInit`               | cloud-init user data config. See cloud-init documentation for more details.                                | `#cloud-config
+| `sshKeys`                 | List of SSH public keys for authentication. Can be a single key or a list of keys.                         | `[]`         |
-` |
+| `cloudInit`               | cloud-init user data config. See cloud-init documentation for more details.                                | `""`         |
+| `cloudInitSeed`           | A seed string to generate an SMBIOS UUID for the VM.                                                       | `""`         |
 
 ## U Series
 

packages/apps/virtual-machine/templates/_helpers.tpl

@@ -49,3 +49,23 @@ Selector labels
 app.kubernetes.io/name: {{ include "virtual-machine.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
+
+{{/*
+Generate a stable UUID for cloud-init re-initialization upon upgrade.
+*/}}
+{{- define "virtual-machine.stableUuid" -}}
+{{- $source := printf "%s-%s-%s" .Release.Namespace (include "virtual-machine.fullname" .) .Values.cloudInitSeed }}
+{{- $hash := sha256sum $source }}
+{{- $uuid := printf "%s-%s-4%s-9%s-%s" (substr 0 8 $hash) (substr 8 12 $hash) (substr 13 16 $hash) (substr 17 20 $hash) (substr 20 32 $hash) }}
+{{- if eq .Values.cloudInitSeed "" }}
+  {{- /*  Try to save previous uuid to not trigger full cloud-init again if user decided to remove the seed. */}}
+  {{- $vmResource := lookup "kubevirt.io/v1" "VirtualMachine" .Release.Namespace (include "virtual-machine.fullname" .) -}}
+  {{- if $vmResource }}
+    {{- $existingUuid := $vmResource | dig "spec" "template" "spec" "domain" "firmware" "uuid" "" }}
+    {{- if $existingUuid }}
+      {{- $uuid = $existingUuid }}
+    {{- end }}
+  {{- end }}
+{{- end }}
+{{- $uuid }}
+{{- end }}

packages/apps/virtual-machine/templates/vm.yaml

@@ -68,7 +68,15 @@ spec:
           requests:
             memory: {{ .Values.resources.memory | quote }}
         {{- end }}
+        firmware:
+          uuid: {{ include "virtual-machine.stableUuid" . }}
         devices:
+          {{- if .Values.gpus }}
+          gpus:
+          {{- range $i, $gpu := .Values.gpus }}
+          - deviceName: {{ $gpu.name }}
+          {{- end }}
+          {{- end }}
           disks:
           - disk:
               bus: scsi
@@ -90,6 +98,7 @@ spec:
             secret:
               secretName: {{ include "virtual-machine.fullname" $ }}-ssh-keys
           propagationMethod:
+            # keys will be injected into metadata part of cloud-init disk
             noCloud: {}
       {{- end }}
       terminationGracePeriodSeconds: 30
@@ -100,8 +109,14 @@ spec:
       {{- if or .Values.sshKeys .Values.cloudInit }}
       - name: cloudinitdisk
         cloudInitNoCloud:
+        {{- if .Values.cloudInit }}
           secretRef:
             name: {{ include "virtual-machine.fullname" . }}-cloud-init
+        {{- else }}
+          userData: |
+            #cloud-config
+            final_message: Cloud-init user-data was left blank intentionally.
+        {{- end }}
       {{- end }}
       networks:
       - name: default

packages/apps/virtual-machine/values.schema.json

@@ -88,7 +88,7 @@
     },
     "instanceProfile": {
       "type": "string",
-      "description": "Virtual Machine prefferences profile",
+      "description": "Virtual Machine preferences profile",
       "default": "ubuntu",
       "optional": true,
       "enum": [
@@ -164,6 +164,14 @@
         }
       }
     },
+    "gpus": {
+      "type": "array",
+      "description": "List of GPUs to attach",
+      "default": [],
+      "items": {
+        "type": "object"
+      }
+    },
     "resources": {
       "type": "object",
       "properties": {
@@ -190,7 +198,12 @@
     "cloudInit": {
       "type": "string",
       "description": "cloud-init user data config. See cloud-init documentation for more details.",
-      "default": "#cloud-config\n"
+      "default": ""
+    },
+    "cloudInitSeed": {
+      "type": "string",
+      "description": "A seed string to generate an SMBIOS UUID for the VM.",
+      "default": ""
     }
   }
 }

packages/apps/virtual-machine/values.yaml

@@ -12,7 +12,7 @@ externalPorts:
 running: true
 
 ## @param instanceType Virtual Machine instance type
-## @param instanceProfile Virtual Machine prefferences profile
+## @param instanceProfile Virtual Machine preferences profile
 ##
 instanceType: "u1.medium"
 instanceProfile: ubuntu
@@ -26,6 +26,12 @@ systemDisk:
   storage: 5Gi
   storageClass: replicated
 
+## @param gpus [array] List of GPUs to attach
+## Example:
+## gpus:
+## - name: nvidia.com/GA102GL_A10
+gpus: []
+
 ## @param resources.cpu The number of CPU cores allocated to the virtual machine
 ## @param resources.memory The amount of memory allocated to the virtual machine
 resources:
@@ -49,5 +55,13 @@ sshKeys: []
 ##   password: ubuntu
 ##   chpasswd: { expire: False }
 ##
-cloudInit: |
+cloudInit: ""
-  #cloud-config
+
+## @param cloudInitSeed A seed string to generate an SMBIOS UUID for the VM.
+cloudInitSeed: ""
+## Change it to any new value to force a full cloud-init reconfiguration. Change it when you want to apply
+## to an existing VM settings that are usually written only once, like new SSH keys or new network configuration.
+## An empty value does nothing (and the existing UUID is not reverted). Please note that changing this value
+## does not trigger a VM restart. You must perform the restart separately.
+## Example:
+## cloudInitSeed: "upd1"

packages/apps/vm-disk/Chart.yaml

@@ -16,10 +16,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: 0.1.0
+appVersion: 0.1.1

packages/apps/vm-disk/templates/dv.yaml

@@ -3,7 +3,9 @@ apiVersion: cdi.kubevirt.io/v1beta1
 kind: DataVolume
 metadata:
   annotations:
+    {{- if hasKey .Values.source "upload" }}
     cdi.kubevirt.io/storage.bind.immediate.requested: ""
+    {{- end }}
     vm-disk.cozystack.io/optical: "{{ .Values.optical }}"
   name: {{ .Release.Name }}
 spec:

packages/apps/vm-instance/Chart.yaml

@@ -17,10 +17,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.1
+version: 0.6.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.5.1"
+appVersion: 0.6.0

packages/apps/vm-instance/Makefile

@@ -3,6 +3,7 @@ include ../../../scripts/package.mk
 generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md
 	yq -o json -i '.properties.disks.items.type = "object" | .properties.disks.default = []' values.schema.json
+	yq -o json -i '.properties.gpus.items.type = "object" | .properties.gpus.default = []' values.schema.json
 	INSTANCE_TYPES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/instancetypes.yaml | yq 'split(" ") | . + [""]' -o json) \
 	  && yq -i -o json ".properties.instanceType.optional=true | .properties.instanceType.enum = $${INSTANCE_TYPES}" values.schema.json
 	PREFERENCES=$$(yq e '.metadata.name' -o=json -r ../../system/kubevirt-instancetypes/templates/preferences.yaml | yq 'split(" ") | . + [""]' -o json) \

packages/apps/vm-instance/README.md

@@ -36,20 +36,21 @@ virtctl ssh <user>@<vm>
 
 ### Common parameters
 
-| Name               | Description                                                                                                | Value            |
+| Name               | Description                                                                                                | Value       |
-| ------------------ | ---------------------------------------------------------------------------------------------------------- | ---------------- |
+| ------------------ | ---------------------------------------------------------------------------------------------------------- | ----------- |
-| `external`         | Enable external access from outside the cluster                                                            | `false`          |
+| `external`         | Enable external access from outside the cluster                                                            | `false`     |
-| `externalMethod`   | specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` | `WholeIP`        |
+| `externalMethod`   | specify method to passthrough the traffic to the virtual machine. Allowed values: `WholeIP` and `PortList` | `WholeIP`   |
-| `externalPorts`    | Specify ports to forward from outside the cluster                                                          | `[]`             |
+| `externalPorts`    | Specify ports to forward from outside the cluster                                                          | `[]`        |
-| `running`          | Determines if the virtual machine should be running                                                        | `true`           |
+| `running`          | Determines if the virtual machine should be running                                                        | `true`      |
-| `instanceType`     | Virtual Machine instance type                                                                              | `u1.medium`      |
+| `instanceType`     | Virtual Machine instance type                                                                              | `u1.medium` |
-| `instanceProfile`  | Virtual Machine prefferences profile                                                                       | `ubuntu`         |
+| `instanceProfile`  | Virtual Machine preferences profile                                                                        | `ubuntu`    |
-| `disks`            | List of disks to attach                                                                                    | `[]`             |
+| `disks`            | List of disks to attach                                                                                    | `[]`        |
-| `resources.cpu`    | The number of CPU cores allocated to the virtual machine                                                   | `""`             |
+| `gpus`             | List of GPUs to attach                                                                                     | `[]`        |
-| `resources.memory` | The amount of memory allocated to the virtual machine                                                      | `""`             |
+| `resources.cpu`    | The number of CPU cores allocated to the virtual machine                                                   | `""`        |
-| `sshKeys`          | List of SSH public keys for authentication. Can be a single key or a list of keys.                         | `[]`             |
+| `resources.memory` | The amount of memory allocated to the virtual machine                                                      | `""`        |
-| `cloudInit`        | cloud-init user data config. See cloud-init documentation for more details.                                | `#cloud-config
+| `sshKeys`          | List of SSH public keys for authentication. Can be a single key or a list of keys.                         | `[]`        |
-` |
+| `cloudInit`        | cloud-init user data config. See cloud-init documentation for more details.                                | `""`        |
+| `cloudInitSeed`    | A seed string to generate an SMBIOS UUID for the VM.                                                       | `""`        |
 
 ## U Series
 

packages/apps/vm-instance/templates/_helpers.tpl

@@ -49,3 +49,23 @@ Selector labels
 app.kubernetes.io/name: {{ include "virtual-machine.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
+
+{{/*
+Generate a stable UUID for cloud-init re-initialization upon upgrade.
+*/}}
+{{- define "virtual-machine.stableUuid" -}}
+{{- $source := printf "%s-%s-%s" .Release.Namespace (include "virtual-machine.fullname" .) .Values.cloudInitSeed }}
+{{- $hash := sha256sum $source }}
+{{- $uuid := printf "%s-%s-4%s-9%s-%s" (substr 0 8 $hash) (substr 8 12 $hash) (substr 13 16 $hash) (substr 17 20 $hash) (substr 20 32 $hash) }}
+{{- if eq .Values.cloudInitSeed "" }}
+  {{- /*  Try to save previous uuid to not trigger full cloud-init again if user decided to remove the seed. */}}
+  {{- $vmResource := lookup "kubevirt.io/v1" "VirtualMachine" .Release.Namespace (include "virtual-machine.fullname" .) -}}
+  {{- if $vmResource }}
+    {{- $existingUuid := $vmResource | dig "spec" "template" "spec" "domain" "firmware" "uuid" "" }}
+    {{- if $existingUuid }}
+      {{- $uuid = $existingUuid }}
+    {{- end }}
+  {{- end }}
+{{- end }}
+{{- $uuid }}
+{{- end }}

packages/apps/vm-instance/templates/dashboard-resourcemap.yaml

@@ -22,5 +22,5 @@ spec:
   kind: virtual-machine
   type: virtual-machine
   selector:
-    vm.kubevirt.io/name: {{ $.Release.Name }}
+    {{- include "virtual-machine.selectorLabels" . | nindent 4 }}
   version: {{ $.Chart.Version }}

packages/apps/vm-instance/templates/vm.yaml

@@ -1,8 +1,8 @@
 {{- if and .Values.instanceType (not (lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" .Values.instanceType)) }}
-{{-   fail (printf "Specified instancetype not exists in cluster: %s" .Values.instanceType) }}
+{{-   fail (printf "Specified instanceType does not exist in the cluster: %s" .Values.instanceType) }}
 {{- end }}
 {{- if and .Values.instanceProfile (not (lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterPreference" "" .Values.instanceProfile)) }}
-{{-   fail (printf "Specified profile not exists in cluster: %s" .Values.instanceProfile) }}
+{{-   fail (printf "Specified instanceProfile does not exist in the cluster: %s" .Values.instanceProfile) }}
 {{- end }}
 
 apiVersion: kubevirt.io/v1
@@ -40,11 +40,19 @@ spec:
           requests:
             memory: {{ .Values.resources.memory | quote }}
         {{- end }}
+        firmware:
+          uuid: {{ include "virtual-machine.stableUuid" . }}
         devices:
+          {{- if .Values.gpus }}
+          gpus:
+          {{- range $i, $gpu := .Values.gpus }}
+          - deviceName: {{ $gpu.name }}
+          {{- end }}
+          {{- end }}
           disks:
           {{- range $i, $disk := .Values.disks }}
-          - name: disk-{{ .name }}
+          - name: disk-{{ $disk.name }}
-            {{- $disk := lookup "cdi.kubevirt.io/v1beta1" "DataVolume" $.Release.Namespace (printf "vm-disk-%s" .name) }}
+            {{- $disk := lookup "cdi.kubevirt.io/v1beta1" "DataVolume" $.Release.Namespace (printf "vm-disk-%s" $disk.name) }}
             {{- if $disk }}
             {{- if and (hasKey $disk.metadata.annotations "vm-disk.cozystack.io/optical") (eq (index $disk.metadata.annotations "vm-disk.cozystack.io/optical") "true") }}
             cdrom: {}
@@ -75,6 +83,7 @@ spec:
             secret:
               secretName: {{ include "virtual-machine.fullname" $ }}-ssh-keys
           propagationMethod:
+            # keys will be injected into metadata part of cloud-init disk
             noCloud: {}
       {{- end }}
       terminationGracePeriodSeconds: 30
@@ -87,8 +96,14 @@ spec:
       {{- if or .Values.sshKeys .Values.cloudInit }}
       - name: cloudinitdisk
         cloudInitNoCloud:
+        {{- if .Values.cloudInit }}
           secretRef:
             name: {{ include "virtual-machine.fullname" . }}-cloud-init
+        {{- else }}
+          userData: |
+            #cloud-config
+            final_message: Cloud-init user-data was left blank intentionally.
+        {{- end }}
       {{- end }}
       networks:
       - name: default

packages/apps/vm-instance/values.schema.json

@@ -88,7 +88,7 @@
     },
     "instanceProfile": {
       "type": "string",
-      "description": "Virtual Machine prefferences profile",
+      "description": "Virtual Machine preferences profile",
       "default": "ubuntu",
       "optional": true,
       "enum": [
@@ -145,6 +145,14 @@
         "type": "object"
       }
     },
+    "gpus": {
+      "type": "array",
+      "description": "List of GPUs to attach",
+      "default": [],
+      "items": {
+        "type": "object"
+      }
+    },
     "resources": {
       "type": "object",
       "properties": {
@@ -171,7 +179,12 @@
     "cloudInit": {
       "type": "string",
       "description": "cloud-init user data config. See cloud-init documentation for more details.",
-      "default": "#cloud-config\n"
+      "default": ""
+    },
+    "cloudInitSeed": {
+      "type": "string",
+      "description": "A seed string to generate an SMBIOS UUID for the VM.",
+      "default": ""
     }
   }
 }

packages/apps/vm-instance/values.yaml

@@ -12,7 +12,7 @@ externalPorts:
 running: true
 
 ## @param instanceType Virtual Machine instance type
-## @param instanceProfile Virtual Machine prefferences profile
+## @param instanceProfile Virtual Machine preferences profile
 ##
 instanceType: "u1.medium"
 instanceProfile: ubuntu
@@ -24,6 +24,12 @@ instanceProfile: ubuntu
 ## - name: example-data
 disks: []
 
+## @param gpus [array] List of GPUs to attach
+## Example:
+## gpus:
+## - name: nvidia.com/GA102GL_A10
+gpus: []
+
 ## @param resources.cpu The number of CPU cores allocated to the virtual machine
 ## @param resources.memory The amount of memory allocated to the virtual machine
 resources:
@@ -47,5 +53,13 @@ sshKeys: []
 ##   password: ubuntu
 ##   chpasswd: { expire: False }
 ##
-cloudInit: |
+cloudInit: ""
-  #cloud-config
+
+## @param cloudInitSeed A seed string to generate an SMBIOS UUID for the VM.
+cloudInitSeed: ""
+## Change it to any new value to force a full cloud-init reconfiguration. Change it when you want to apply
+## to an existing VM settings that are usually written only once, like new SSH keys or new network configuration.
+## An empty value does nothing (and the existing UUID is not reverted). Please note that changing this value
+## does not trigger a VM restart. You must perform the restart separately.
+## Example:
+## cloudInitSeed: "upd1"

packages/core/builder/Makefile

@@ -1,35 +0,0 @@
-NAMESPACE=cozy-builder
-NAME := builder
-
-TALOS_VERSION=$(shell awk '/^version:/ {print $$2}' ../installer/images/talos/profiles/installer.yaml)
-
-include ../../../scripts/common-envs.mk
-
-help: ## Show this help.
-	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
-
-show:
-	helm template -n $(NAMESPACE) $(NAME) .
-
-apply: ## Create builder sandbox in existing Kubernetes cluster.
-	helm template -n $(NAMESPACE) $(NAME) . | kubectl apply -f -
-	docker buildx ls | grep -q '^buildkit-builder*' || docker buildx create \
-		--bootstrap \
-		--name=buildkit-$(NAME) \
-		--driver=kubernetes \
-		--driver-opt=namespace=$(NAMESPACE),replicas=1 \
-		--platform=linux/amd64 \
-		--platform=linux/arm64 \
-		--use \
-		--config config.toml
-
-diff:
-	helm template -n $(NAMESPACE) $(NAME) . | kubectl diff -f -
-
-delete: ## Remove builder sandbox from existing Kubernetes cluster.
-	kubectl delete deploy -n $(NAMESPACE) $(NAME)-talos-imager
-	docker buildx rm buildkit-$(NAME)
-
-wait-for-builder:
-	kubectl wait deploy --for=condition=Progressing -n $(NAMESPACE) $(NAME)-talos-imager
-	kubectl wait pod --for=condition=Ready -n $(NAMESPACE) -l app=$(NAME)-talos-imager

packages/core/builder/config.toml

@@ -1,11 +0,0 @@
-[worker.oci]
-  gc = true
-  gckeepstorage = 50000
-
-  [[worker.oci.gcpolicy]]
-    keepBytes = 10737418240
-    keepDuration = 604800
-    filters = [ "type==source.local", "type==exec.cachemount", "type==source.git.checkout"]
-  [[worker.oci.gcpolicy]]
-    all = true
-    keepBytes = 53687091200

packages/core/builder/templates/sandbox.yaml

@@ -1,43 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: {{ .Release.Namespace }}
-  labels:
-    pod-security.kubernetes.io/enforce: privileged
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ .Release.Name }}-talos-imager
-  namespace: {{ .Release.Namespace }}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: {{ .Release.Name }}-talos-imager
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        app: {{ .Release.Name }}-talos-imager
-    spec:
-      automountServiceAccountToken: false
-      terminationGracePeriodSeconds: 1
-      containers:
-      - name: imager
-        image: "{{ .Values.talos.imager.image }}"
-        securityContext:
-          privileged: true
-        command:
-        - sleep
-        - infinity
-        volumeMounts:
-        - mountPath: /dev
-          name: dev
-      volumes:
-      - hostPath:
-          path: /dev
-          type: Directory
-        name: dev

packages/core/builder/values.yaml

@@ -1,3 +0,0 @@
-talos:
-  imager:
-    image: ghcr.io/siderolabs/imager:v1.9.3

packages/core/installer/Makefile

@@ -19,12 +19,10 @@ diff:
 
 update:
 	hack/gen-profiles.sh
-	IMAGE=$$(yq '.input.baseInstaller.imageRef | sub("/installer:", "/imager:")' images/talos/profiles/installer.yaml) \
-		yq -i '.talos.imager.image = strenv(IMAGE)' ../builder/values.yaml
 
-image: pre-checks image-cozystack image-talos image-matchbox
+image: pre-checks image-matchbox image-cozystack image-talos
 
-image-cozystack: run-builder
+image-cozystack:
 	make -C ../../.. repos
 	docker buildx build -f images/cozystack/Dockerfile ../../.. \
 		--provenance false \
@@ -40,11 +38,11 @@ image-cozystack: run-builder
 		yq -i '.cozystack.image = strenv(IMAGE)' values.yaml
 	rm -f images/installer.json
 
-image-talos: run-builder
+image-talos:
 	test -f ../../../_out/assets/installer-amd64.tar || make talos-installer
 	skopeo copy docker-archive:../../../_out/assets/installer-amd64.tar docker://$(REGISTRY)/talos:$(call settag,$(TALOS_VERSION))
 
-image-matchbox: run-builder
+image-matchbox:
 	test -f ../../../_out/assets/kernel-amd64 || make talos-kernel
 	test -f ../../../_out/assets/initramfs-metal-amd64.xz || make talos-initramfs
 	docker buildx build -f images/matchbox/Dockerfile ../../.. \
@@ -61,13 +59,10 @@ image-matchbox: run-builder
 		> ../../extra/bootbox/images/matchbox.tag
 	rm -f images/matchbox.json
 
-assets: talos-iso talos-nocloud talos-metal
+assets: talos-iso talos-nocloud talos-metal talos-kernel talos-initramfs
 
 talos-initramfs talos-kernel talos-installer talos-iso talos-nocloud talos-metal:
 	mkdir -p ../../../_out/assets
 	cat images/talos/profiles/$(subst talos-,,$@).yaml | \
-		kubectl exec -i -n cozy-builder deploy/builder-talos-imager -- imager --tar-to-stdout - | \
+		docker run --rm -i -v /dev:/dev --privileged "ghcr.io/siderolabs/imager:$(TALOS_VERSION)" --tar-to-stdout - | \
 		tar -C ../../../_out/assets -xzf-
-
-run-builder:
-	make -C ../builder/ apply wait-for-builder

packages/core/installer/images/cozystack/Dockerfile

@@ -30,6 +30,8 @@ FROM alpine:3.21
 
 RUN apk add --no-cache make
 RUN apk add helm kubectl --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
+RUN apk add yq
+RUN apk add coreutils
 
 COPY scripts /cozystack/scripts
 COPY --from=builder /src/packages/core /cozystack/packages/core

packages/core/installer/images/talos/profiles/initramfs.yaml

@@ -3,24 +3,24 @@
 arch: amd64
 platform: metal
 secureboot: false
-version: v1.9.3
+version: v1.9.5
 input:
   kernel:
     path: /usr/install/amd64/vmlinuz
   initramfs:
     path: /usr/install/amd64/initramfs.xz
   baseInstaller:
-    imageRef: ghcr.io/siderolabs/installer:v1.9.3
+    imageRef: ghcr.io/siderolabs/installer:v1.9.5
   systemExtensions:
-    - imageRef: ghcr.io/siderolabs/amd-ucode:20250109
+    - imageRef: ghcr.io/siderolabs/amd-ucode:20250311
     - imageRef: ghcr.io/siderolabs/amdgpu-firmware:20241110
-    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250109
+    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250311
     - imageRef: ghcr.io/siderolabs/i915-ucode:20241110
-    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/intel-ucode:20241112
+    - imageRef: ghcr.io/siderolabs/intel-ucode:20250211
-    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.3
+    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.5
-    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.3
+    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.5
 output:
   kind: initramfs
   imageOptions: {}

packages/core/installer/images/talos/profiles/installer.yaml

@@ -3,24 +3,24 @@
 arch: amd64
 platform: metal
 secureboot: false
-version: v1.9.3
+version: v1.9.5
 input:
   kernel:
     path: /usr/install/amd64/vmlinuz
   initramfs:
     path: /usr/install/amd64/initramfs.xz
   baseInstaller:
-    imageRef: ghcr.io/siderolabs/installer:v1.9.3
+    imageRef: ghcr.io/siderolabs/installer:v1.9.5
   systemExtensions:
-    - imageRef: ghcr.io/siderolabs/amd-ucode:20250109
+    - imageRef: ghcr.io/siderolabs/amd-ucode:20250311
     - imageRef: ghcr.io/siderolabs/amdgpu-firmware:20241110
-    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250109
+    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250311
     - imageRef: ghcr.io/siderolabs/i915-ucode:20241110
-    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/intel-ucode:20241112
+    - imageRef: ghcr.io/siderolabs/intel-ucode:20250211
-    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.3
+    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.5
-    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.3
+    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.5
 output:
   kind: installer
   imageOptions: {}

packages/core/installer/images/talos/profiles/iso.yaml

@@ -3,24 +3,24 @@
 arch: amd64
 platform: metal
 secureboot: false
-version: v1.9.3
+version: v1.9.5
 input:
   kernel:
     path: /usr/install/amd64/vmlinuz
   initramfs:
     path: /usr/install/amd64/initramfs.xz
   baseInstaller:
-    imageRef: ghcr.io/siderolabs/installer:v1.9.3
+    imageRef: ghcr.io/siderolabs/installer:v1.9.5
   systemExtensions:
-    - imageRef: ghcr.io/siderolabs/amd-ucode:20250109
+    - imageRef: ghcr.io/siderolabs/amd-ucode:20250311
     - imageRef: ghcr.io/siderolabs/amdgpu-firmware:20241110
-    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250109
+    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250311
     - imageRef: ghcr.io/siderolabs/i915-ucode:20241110
-    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/intel-ucode:20241112
+    - imageRef: ghcr.io/siderolabs/intel-ucode:20250211
-    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.3
+    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.5
-    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.3
+    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.5
 output:
   kind: iso
   imageOptions: {}

packages/core/installer/images/talos/profiles/kernel.yaml

@@ -3,24 +3,24 @@
 arch: amd64
 platform: metal
 secureboot: false
-version: v1.9.3
+version: v1.9.5
 input:
   kernel:
     path: /usr/install/amd64/vmlinuz
   initramfs:
     path: /usr/install/amd64/initramfs.xz
   baseInstaller:
-    imageRef: ghcr.io/siderolabs/installer:v1.9.3
+    imageRef: ghcr.io/siderolabs/installer:v1.9.5
   systemExtensions:
-    - imageRef: ghcr.io/siderolabs/amd-ucode:20250109
+    - imageRef: ghcr.io/siderolabs/amd-ucode:20250311
     - imageRef: ghcr.io/siderolabs/amdgpu-firmware:20241110
-    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250109
+    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250311
     - imageRef: ghcr.io/siderolabs/i915-ucode:20241110
-    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/intel-ucode:20241112
+    - imageRef: ghcr.io/siderolabs/intel-ucode:20250211
-    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.3
+    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.5
-    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.3
+    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.5
 output:
   kind: kernel
   imageOptions: {}

packages/core/installer/images/talos/profiles/metal.yaml

@@ -3,24 +3,24 @@
 arch: amd64
 platform: metal
 secureboot: false
-version: v1.9.3
+version: v1.9.5
 input:
   kernel:
     path: /usr/install/amd64/vmlinuz
   initramfs:
     path: /usr/install/amd64/initramfs.xz
   baseInstaller:
-    imageRef: ghcr.io/siderolabs/installer:v1.9.3
+    imageRef: ghcr.io/siderolabs/installer:v1.9.5
   systemExtensions:
-    - imageRef: ghcr.io/siderolabs/amd-ucode:20250109
+    - imageRef: ghcr.io/siderolabs/amd-ucode:20250311
     - imageRef: ghcr.io/siderolabs/amdgpu-firmware:20241110
-    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250109
+    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250311
     - imageRef: ghcr.io/siderolabs/i915-ucode:20241110
-    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/intel-ucode:20241112
+    - imageRef: ghcr.io/siderolabs/intel-ucode:20250211
-    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.3
+    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.5
-    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.3
+    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.5
 output:
   kind: image
   imageOptions: { diskSize: 1306525696, diskFormat: raw }

packages/core/installer/images/talos/profiles/nocloud.yaml

@@ -3,24 +3,24 @@
 arch: amd64
 platform: nocloud
 secureboot: false
-version: v1.9.3
+version: v1.9.5
 input:
   kernel:
     path: /usr/install/amd64/vmlinuz
   initramfs:
     path: /usr/install/amd64/initramfs.xz
   baseInstaller:
-    imageRef: ghcr.io/siderolabs/installer:v1.9.3
+    imageRef: ghcr.io/siderolabs/installer:v1.9.5
   systemExtensions:
-    - imageRef: ghcr.io/siderolabs/amd-ucode:20250109
+    - imageRef: ghcr.io/siderolabs/amd-ucode:20250311
     - imageRef: ghcr.io/siderolabs/amdgpu-firmware:20241110
-    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250109
+    - imageRef: ghcr.io/siderolabs/bnx2-bnx2x:20250311
     - imageRef: ghcr.io/siderolabs/i915-ucode:20241110
-    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/intel-ice-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/intel-ucode:20241112
+    - imageRef: ghcr.io/siderolabs/intel-ucode:20250211
-    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250109
+    - imageRef: ghcr.io/siderolabs/qlogic-firmware:20250311
-    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.3
+    - imageRef: ghcr.io/siderolabs/drbd:9.2.12-v1.9.5
-    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.3
+    - imageRef: ghcr.io/siderolabs/zfs:2.2.7-v1.9.5
 output:
   kind: image
   imageOptions: { diskSize: 1306525696, diskFormat: raw }

packages/core/installer/values.yaml

@@ -1,2 +1,2 @@
 cozystack:
-  image: ghcr.io/cozystack/cozystack/installer:v0.28.0@sha256:71ae2037ca44d49bbcf8be56c127ee92f2486089a8ea1cdd6508af49705956ac
+  image: ghcr.io/cozystack/cozystack/installer:v0.30.2@sha256:59996588b5d59b5593fb34442b2f2ed8ef466d138b229a8d37beb6f70141a690

packages/core/platform/Makefile

@@ -7,7 +7,11 @@ show:
 	helm template -n $(NAMESPACE) $(NAME) . --dry-run=server $(API_VERSIONS_FLAGS)
 
 apply:
-	helm template -n $(NAMESPACE) $(NAME) . --dry-run=server $(API_VERSIONS_FLAGS) | kubectl apply -f-
+	helm template -n $(NAMESPACE) $(NAME) . --dry-run=server $(API_VERSIONS_FLAGS) \
+		| kubectl apply -f-
+	kubectl delete helmreleases.helm.toolkit.fluxcd.io -l cozystack.io/marked-for-deletion=true -A
+
+reconcile: apply
 
 namespaces-show:
 	helm template -n $(NAMESPACE) $(NAME) . --dry-run=server $(API_VERSIONS_FLAGS) -s templates/namespaces.yaml

packages/core/platform/bundles/distro-full.yaml

@@ -31,6 +31,13 @@ releases:
       autoDirectNodeRoutes: true
       routingMode: native
 
+- name: cilium-networkpolicy
+  releaseName: cilium-networkpolicy
+  chart: cozy-cilium-networkpolicy
+  namespace: cozy-cilium
+  privileged: true
+  dependsOn: [cilium]
+
 - name: cozy-proxy
   releaseName: cozystack
   chart: cozy-cozy-proxy
@@ -127,14 +134,14 @@ releases:
   chart: cozy-kafka-operator
   namespace: cozy-kafka-operator
   optional: true
-  dependsOn: [cilium]
+  dependsOn: [cilium,victoria-metrics-operator]
 
 - name: clickhouse-operator
   releaseName: clickhouse-operator
   chart: cozy-clickhouse-operator
   namespace: cozy-clickhouse-operator
   optional: true
-  dependsOn: [cilium]
+  dependsOn: [cilium,victoria-metrics-operator]
 
 - name: rabbitmq-operator
   releaseName: rabbitmq-operator
@@ -154,7 +161,7 @@ releases:
   releaseName: piraeus-operator
   chart: cozy-piraeus-operator
   namespace: cozy-linstor
-  dependsOn: [cilium,cert-manager]
+  dependsOn: [cilium,cert-manager,victoria-metrics-operator]
 
 - name: snapshot-controller
   releaseName: snapshot-controller

packages/core/platform/bundles/distro-hosted.yaml

@@ -96,14 +96,14 @@ releases:
   chart: cozy-kafka-operator
   namespace: cozy-kafka-operator
   optional: true
-  dependsOn: []
+  dependsOn: [victoria-metrics-operator]
 
 - name: clickhouse-operator
   releaseName: clickhouse-operator
   chart: cozy-clickhouse-operator
   namespace: cozy-clickhouse-operator
   optional: true
-  dependsOn: []
+  dependsOn: [victoria-metrics-operator]
 
 - name: rabbitmq-operator
   releaseName: rabbitmq-operator

packages/core/platform/bundles/paas-full.yaml

@@ -34,6 +34,13 @@ releases:
   - values-talos.yaml
   - values-kubeovn.yaml
 
+- name: cilium-networkpolicy
+  releaseName: cilium-networkpolicy
+  chart: cozy-cilium-networkpolicy
+  namespace: cozy-cilium
+  privileged: true
+  dependsOn: [cilium]
+
 - name: kubeovn
   releaseName: kubeovn
   chart: cozy-kubeovn
@@ -109,7 +116,7 @@ releases:
   chart: cozy-monitoring-agents
   namespace: cozy-monitoring
   privileged: true
-  dependsOn: [cilium,kubeovn,victoria-metrics-operator]
+  dependsOn: [victoria-metrics-operator, vertical-pod-autoscaler-crds]
   values:
     scrapeRules:
       etcd:
@@ -146,6 +153,17 @@ releases:
   namespace: cozy-kubevirt-cdi
   dependsOn: [cilium,kubeovn,kubevirt-cdi-operator]
 
+- name: gpu-operator
+  releaseName: gpu-operator
+  chart: cozy-gpu-operator
+  namespace: cozy-gpu-operator
+  privileged: true
+  optional: true
+  dependsOn: [cilium,kubeovn]
+  valuesFiles:
+  - values.yaml
+  - values-talos.yaml
+
 - name: metallb
   releaseName: metallb
   chart: cozy-metallb
@@ -181,13 +199,13 @@ releases:
   releaseName: kafka-operator
   chart: cozy-kafka-operator
   namespace: cozy-kafka-operator
-  dependsOn: [cilium,kubeovn]
+  dependsOn: [cilium,kubeovn,victoria-metrics-operator]
 
 - name: clickhouse-operator
   releaseName: clickhouse-operator
   chart: cozy-clickhouse-operator
   namespace: cozy-clickhouse-operator
-  dependsOn: [cilium,kubeovn]
+  dependsOn: [cilium,kubeovn,victoria-metrics-operator]
 
 - name: rabbitmq-operator
   releaseName: rabbitmq-operator
@@ -252,7 +270,10 @@ releases:
             {{- end }}
     {{- end }}
     {{- end }}
+      frontend:
+        resourcesPreset: "none"
       dashboard:
+        resourcesPreset: "none"
         {{- $cozystackBranding:= lookup "v1" "ConfigMap" "cozy-system" "cozystack-branding" }}
         {{- $branding := dig "data" "branding" "" $cozystackBranding }}
         {{- if $branding }}
@@ -381,6 +402,13 @@ releases:
   privileged: true
   dependsOn: [monitoring-agents]
 
+- name: vertical-pod-autoscaler-crds
+  releaseName: vertical-pod-autoscaler-crds
+  chart: cozy-vertical-pod-autoscaler-crds
+  namespace: cozy-vertical-pod-autoscaler
+  privileged: true
+  dependsOn: [cilium, kubeovn]
+
 - name: reloader
   releaseName: reloader
   chart: cozy-reloader

packages/core/platform/bundles/paas-hosted.yaml

@@ -69,7 +69,7 @@ releases:
   chart: cozy-monitoring-agents
   namespace: cozy-monitoring
   privileged: true
-  dependsOn: [victoria-metrics-operator]
+  dependsOn: [victoria-metrics-operator, vertical-pod-autoscaler-crds]
   values:
     scrapeRules:
       etcd:
@@ -103,13 +103,13 @@ releases:
   releaseName: kafka-operator
   chart: cozy-kafka-operator
   namespace: cozy-kafka-operator
-  dependsOn: []
+  dependsOn: [victoria-metrics-operator]
 
 - name: clickhouse-operator
   releaseName: clickhouse-operator
   chart: cozy-clickhouse-operator
   namespace: cozy-clickhouse-operator
-  dependsOn: []
+  dependsOn: [victoria-metrics-operator]
 
 - name: rabbitmq-operator
   releaseName: rabbitmq-operator
@@ -168,7 +168,10 @@ releases:
             {{- end }}
     {{- end }}
     {{- end }}
+      frontend:
+        resourcesPreset: "none"
       dashboard:
+        resourcesPreset: "none"
         {{- $cozystackBranding:= lookup "v1" "ConfigMap" "cozy-system" "cozystack-branding" }}
         {{- $branding := dig "data" "branding" "" $cozystackBranding }}
         {{- if $branding }}
@@ -254,3 +257,10 @@ releases:
   namespace: cozy-vertical-pod-autoscaler
   privileged: true
   dependsOn: [monitoring-agents]
+
+- name: vertical-pod-autoscaler-crds
+  releaseName: vertical-pod-autoscaler-crds
+  chart: cozy-vertical-pod-autoscaler-crds
+  namespace: cozy-vertical-pod-autoscaler
+  privileged: true
+  dependsOn: [cilium, kubeovn]

packages/core/platform/templates/apps.yaml

@@ -54,6 +54,12 @@ spec:
         namespace: cozy-public
   values:
     host: "{{ $host }}"
+  valuesFrom:
+  - kind: ConfigMap
+    name: "cozy-system-configuration-hash"
+    valuesKey: "cozyTenantConfigurationHash"
+    targetPath: "cozyTenantConfigurationHash"
+    optional: true
   dependsOn:
   {{- range $x := $bundle.releases }}
   {{- if has $x.name (list "cilium" "kubeovn") }}

packages/core/platform/templates/configuration-hash.yaml

@@ -0,0 +1,14 @@
+{{- $rootTenantConfiguration := dict "values" .Values }}
+{{- $cozyConfig     := index (lookup "v1" "ConfigMap" "cozy-system" "cozystack"           ) "data" }}
+{{- $cozyScheduling := index (lookup "v1" "ConfigMap" "cozy-system" "cozystack-scheduling") "data" }}
+{{- $cozyBranding   := index (lookup "v1" "ConfigMap" "cozy-system" "cozystack-branding"  ) "data" }}
+{{- $_ := set $rootTenantConfiguration "config" $cozyConfig }}
+{{- $_ := set $rootTenantConfiguration "scheduling" $cozyScheduling }}
+{{- $_ := set $rootTenantConfiguration "branding" $cozyBranding }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cozy-system-configuration-hash
+  namespace: tenant-root
+data:
+  cozyTenantConfigurationHash: {{ sha256sum (toJson $rootTenantConfiguration) | quote }}

packages/core/platform/templates/helmreleases.yaml

@@ -7,12 +7,23 @@
 
 {{/* collect dependency namespaces from releases */}}
 {{- range $x := $bundle.releases }}
-{{- $_ := set $dependencyNamespaces $x.name $x.namespace }}
+{{-   $_ := set $dependencyNamespaces $x.name $x.namespace }}
 {{- end }}
 
 {{- range $x := $bundle.releases }}
-{{- if not (has $x.name $disabledComponents) }}
+
-{{- if or (not $x.optional) (and ($x.optional) (has $x.name $enabledComponents)) }}
+{{- $shouldInstall := true }}
+{{- $shouldDelete := false }}
+{{- if or (has $x.name $disabledComponents) (and ($x.optional) (not (has $x.name $enabledComponents))) }}
+{{-   $shouldInstall = false }}
+{{-   if $.Capabilities.APIVersions.Has "helm.toolkit.fluxcd.io/v2" }}
+{{-     if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" $x.namespace $x.name }}
+{{-       $shouldDelete = true }}
+{{-     end }}
+{{-   end }}
+{{- end }}
+
+{{- if or $shouldInstall $shouldDelete }}
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
@@ -22,6 +33,9 @@ metadata:
   labels:
     cozystack.io/repository: system
     cozystack.io/system-app: "true"
+    {{- if $shouldDelete }}
+    cozystack.io/marked-for-deletion: "true"
+    {{- end }}
 spec:
   interval: 5m
   releaseName: {{ $x.releaseName | default $x.name }}
@@ -47,10 +61,10 @@ spec:
       {{- end }}
   {{- $values := dict }}
   {{- with $x.values }}
-  {{- $values = merge . $values }}
+  {{-   $values = merge . $values }}
   {{- end }}
   {{- with index $cozyConfig.data (printf "values-%s" $x.name) }}
-  {{- $values = merge (fromYaml .) $values }}
+  {{-   $values = merge (fromYaml .) $values }}
   {{- end }}
   {{- with $values }}
   values:
@@ -70,13 +84,12 @@ spec:
 
   {{- with $x.dependsOn }}
   dependsOn:
-  {{- range $dep := . }}
+  {{-   range $dep := . }}
-  {{- if not (has $dep $disabledComponents) }}
+  {{-     if not (has $dep $disabledComponents) }}
   - name: {{ $dep }}
     namespace: {{ index $dependencyNamespaces $dep }}
-  {{- end }}
+  {{-     end }}
-  {{- end }}
+  {{-   end }}
   {{- end }}
 {{- end }}
 {{- end }}
-{{- end }}

packages/core/testing/Makefile

@@ -2,6 +2,9 @@ NAMESPACE=cozy-e2e-tests
 NAME := sandbox
 CLEAN := 1
 TESTING_APPS := $(shell find ../../apps -maxdepth 1 -mindepth 1 -type d | awk -F/ '{print $$NF}')
+SANDBOX_NAME := cozy-e2e-sandbox-$(shell echo "$$(hostname):$$(pwd)" | sha256sum | cut -c -6)
+
+ROOT_DIR = $(dir $(abspath $(firstword $(MAKEFILE_LIST))/../../..))
 
 include ../../../scripts/common-envs.mk
 
@@ -24,7 +27,6 @@ image-e2e-sandbox:
 		--provenance false \
 		--tag $(REGISTRY)/e2e-sandbox:$(call settag,$(TAG)) \
 		--cache-from type=registry,ref=$(REGISTRY)/e2e-sandbox:latest \
-		--platform linux/amd64,linux/arm64 \
 		--cache-to type=inline \
 		--metadata-file images/e2e-sandbox.json \
 		--push=$(PUSH) \
@@ -34,27 +36,20 @@ image-e2e-sandbox:
 		yq -i '.e2e.image = strenv(IMAGE)' values.yaml
 	rm -f images/e2e-sandbox.json
 
-copy-hack-dir:
+test: ## Run the end-to-end tests in existing sandbox.
-	tar -C ../../../ -cf- hack | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- tar -xf-
+	docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && export COZYSTACK_INSTALLER_YAML=$$(helm template -n cozy-system installer ./packages/core/installer) && hack/e2e.sh'
 
-copy-image:
+test-applications: ## Run the end-to-end tests in existing sandbox for applications.
-	cat ../../../_out/assets/nocloud-amd64.raw.xz | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -xec 'xz --decompress > /nocloud-amd64.raw'
-
-test: wait-for-sandbox copy-hack-dir copy-image ## Run the end-to-end tests in existing sandbox.
-	helm template -n cozy-system installer ../installer | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'cat > /cozystack-installer.yaml'
-	kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'export COZYSTACK_INSTALLER_YAML=$$(cat /cozystack-installer.yaml) && /hack/e2e.sh'
-
-test-applications: wait-for-sandbox copy-hack-dir ## Run the end-to-end tests in existing sandbox for applications.
 	for app in $(TESTING_APPS); do \
-		kubectl exec -ti -n cozy-e2e-tests deploy/cozystack-e2e-sandbox -- bash -c "/hack/e2e.application.sh $${app}"; \
+		docker exec ${SANDBOX_NAME} bash -c "/hack/e2e.application.sh $${app}"; \
 	done
-	kubectl exec -ti -n cozy-e2e-tests deploy/cozystack-e2e-sandbox -- bash -c "kubectl get hr -A | grep -v 'True'"
+	docker exec  ${SANDBOX_NAME} bash -c "kubectl get hr -A | grep -v 'True'"
 
 delete: ## Remove sandbox from existing Kubernetes cluster.
-	kubectl delete deploy -n $(NAMESPACE) cozystack-e2e-$(NAME)
+	docker rm -f "${SANDBOX_NAME}" || true
 
 exec:  ## Opens an interactive shell in the sandbox container.
-	kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- bash
+	docker exec -ti "${SANDBOX_NAME}" -- bash
 
 proxy: sync-hosts ## Enable a SOCKS5 proxy server; mirrord and gost must be installed.
 	mirrord exec --target deploy/cozystack-e2e-sandbox --target-namespace cozy-e2e-tests -- gost -L=127.0.0.1:10080
@@ -65,6 +60,6 @@ login: ## Downloads the kubeconfig into a temporary directory and runs a shell w
 sync-hosts:
 	kubectl exec -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'kubectl get ing -A -o go-template='\''{{ "127.0.0.1 localhost\n"}}{{ range .items }}{{ range .status.loadBalancer.ingress }}{{ .ip }}{{ end }} {{ range .spec.rules }}{{ .host }}{{ end }}{{ "\n" }}{{ end }}'\'' > /etc/hosts'
 
-wait-for-sandbox:
+apply: delete
-	kubectl wait deploy --for=condition=Progressing -n $(NAMESPACE) cozystack-e2e-$(NAME)
+	docker run -d --rm --name "${SANDBOX_NAME}" --privileged "$$(yq .e2e.image values.yaml)" sleep infinity
-	kubectl wait pod --for=condition=Ready -n $(NAMESPACE) -l app=cozystack-e2e-$(NAME)
+	docker cp "${ROOT_DIR}" "${SANDBOX_NAME}":/workspace

packages/core/testing/images/e2e-sandbox/Dockerfile

@@ -1,11 +1,11 @@
 FROM ubuntu:22.04
 
 ARG KUBECTL_VERSION=1.32.0
-ARG TALOSCTL_VERSION=1.8.4
+ARG TALOSCTL_VERSION=1.9.5
 ARG HELM_VERSION=3.16.4
 
 RUN apt-get update
-RUN apt-get -y install genisoimage qemu-kvm qemu-utils iproute2 iptables wget xz-utils netcat curl jq
+RUN apt-get -y install genisoimage qemu-kvm qemu-utils iproute2 iptables wget xz-utils netcat curl jq make git
 RUN curl -LO "https://github.com/siderolabs/talos/releases/download/v${TALOSCTL_VERSION}/talosctl-linux-amd64" \
  && chmod +x talosctl-linux-amd64 \
  && mv talosctl-linux-amd64 /usr/local/bin/talosctl
@@ -14,3 +14,4 @@ RUN curl -LO "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kube
  && mv kubectl /usr/local/bin/kubectl
 RUN curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -s - --version "v${HELM_VERSION}"
 RUN wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O /usr/local/bin/yq && chmod +x /usr/local/bin/yq
+RUN curl -s https://fluxcd.io/install.sh | bash

packages/core/testing/templates/sandbox.yaml

@@ -1,40 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: {{ .Release.Namespace }}
-  labels:
-    pod-security.kubernetes.io/enforce: privileged
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: cozystack-e2e-{{ .Release.Name }}
-  namespace: cozy-e2e-tests
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: cozystack-e2e-{{ .Release.Name }}
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        app: cozystack-e2e-{{ .Release.Name }}
-    spec:
-      automountServiceAccountToken: false
-      terminationGracePeriodSeconds: 1
-      containers:
-      - name: sandbox
-        image: "{{ .Values.e2e.image }}"
-        securityContext:
-          privileged: true
-        env:
-        - name: KUBECONFIG
-          value: /kubeconfig
-        - name: TALOSCONFIG
-          value: /talosconfig
-        command:
-        - sleep
-        - infinity

packages/core/testing/values.yaml

@@ -1,2 +1,2 @@
 e2e:
-  image: ghcr.io/cozystack/cozystack/e2e-sandbox:v0.28.0@sha256:bb5e8f5d92e2e4305ea1cc7f007b3e98769645ab845f632b4788b9373cd207eb
+  image: ghcr.io/cozystack/cozystack/e2e-sandbox:v0.30.2@sha256:31273d6b42dc88c2be2ff9ba64564d1b12e70ae8a5480953341b0d113ac7d4bd

packages/extra/bootbox/images/matchbox.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/matchbox:v0.28.0@sha256:b2002815727b71e2657a6f5b8ed558cc38fc21e81a39b9699266e558be03561f
+ghcr.io/cozystack/cozystack/matchbox:v0.30.2@sha256:307d382f75f1dcb39820c73b93b2ce576cdb6d58032679bda7d926999c677900

packages/extra/monitoring/Chart.yaml

@@ -3,4 +3,4 @@ name: monitoring
 description: Monitoring and observability stack
 icon: /logos/monitoring.svg
 type: application
-version: 1.9.1
+version: 1.9.2

packages/extra/monitoring/images/grafana.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/grafana:1.9.0@sha256:a492931b49af55ad184b485bcd7ea06f1334722d2184702d9f6f2e4123032357
+ghcr.io/cozystack/cozystack/grafana:1.9.2@sha256:c63978e1ed0304e8518b31ddee56c4e8115541b997d8efbe1c0a74da57140399

packages/extra/monitoring/templates/vlogs/vlogs.yaml

@@ -4,6 +4,8 @@ kind: VLogs
 metadata:
   name: {{ .name }}
 spec:
+  image:
+    tag: v1.17.0-victorialogs
   storage:
     resources:
       requests:

packages/extra/versions_map

@@ -34,7 +34,8 @@ monitoring 1.7.0 2a976afe
 monitoring 1.8.0 8c460528
 monitoring 1.8.1 8267072d
 monitoring 1.9.0 45a7416c
-monitoring 1.9.1 HEAD
+monitoring 1.9.1 fd240701
+monitoring 1.9.2 HEAD
 seaweedfs 0.1.0 71514249
 seaweedfs 0.2.0 5fb9cfe3
 seaweedfs 0.2.1 fde4bcfa

packages/system/bucket/images/s3manager.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:218d0c017ae556e5afd074366d9a3124f954c5aefc6474844942420cca8b7640
+ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:a47d2743d01bff0ce60aa745fdff54f9b7184dff8679b11ab4ecd08ac663012b

packages/system/capi-operator/charts/cluster-api-operator/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
-appVersion: 0.17.0
+appVersion: 0.18.1
 description: Cluster API Operator
 name: cluster-api-operator
 type: application
-version: 0.17.0
+version: 0.18.1

packages/system/capi-operator/charts/cluster-api-operator/templates/addon.yaml

@@ -26,8 +26,10 @@ apiVersion: v1
 kind: Namespace
 metadata:
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "1"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "1"
   name: {{ $addonNamespace }}
 ---
@@ -37,8 +39,10 @@ metadata:
   name: {{ $addonName }}
   namespace: {{ $addonNamespace }}
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "2"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "2"
 {{- if or $addonVersion $.Values.secretName }}
 spec:

packages/system/capi-operator/charts/cluster-api-operator/templates/bootstrap.yaml

@@ -26,8 +26,11 @@ apiVersion: v1
 kind: Namespace
 metadata:
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "1"
+    {{- end }}
+    "argocd.argoproj.io/sync-wave": "1"
   name: {{ $bootstrapNamespace }}
 ---
 apiVersion: operator.cluster.x-k8s.io/v1alpha2
@@ -36,8 +39,11 @@ metadata:
   name: {{ $bootstrapName }}
   namespace: {{ $bootstrapNamespace }}
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "2"
+    {{- end }}
+    "argocd.argoproj.io/sync-wave": "2"
 {{- if or $bootstrapVersion $.Values.configSecret.name }}
 spec:
 {{- end}}

packages/system/capi-operator/charts/cluster-api-operator/templates/control-plane.yaml

@@ -26,8 +26,11 @@ apiVersion: v1
 kind: Namespace
 metadata:
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "1"
+    {{- end }}
+    "argocd.argoproj.io/sync-wave": "1"
   name: {{ $controlPlaneNamespace }}
 ---
 apiVersion: operator.cluster.x-k8s.io/v1alpha2
@@ -36,8 +39,11 @@ metadata:
   name: {{ $controlPlaneName }}
   namespace: {{ $controlPlaneNamespace }}
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "2"
+    {{- end }}
+    "argocd.argoproj.io/sync-wave": "2"
 {{- if or $controlPlaneVersion $.Values.configSecret.name $.Values.manager }}
 spec:
 {{- end}}

packages/system/capi-operator/charts/cluster-api-operator/templates/core-conditions.yaml

@@ -1,4 +1,4 @@
-{{- if or .Values.addon .Values.bootstrap .Values.controlPlane .Values.infrastructure }}
+{{- if or .Values.addon .Values.bootstrap .Values.controlPlane .Values.infrastructure .Values.ipam }}
 # Deploy core components if not specified
 {{- if not .Values.core }}
 ---
@@ -6,8 +6,11 @@ apiVersion: v1
 kind: Namespace
 metadata:
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "1"
+    {{- end }}
+    "argocd.argoproj.io/sync-wave": "1"
   name: capi-system
 ---
 apiVersion: operator.cluster.x-k8s.io/v1alpha2
@@ -16,8 +19,11 @@ metadata:
   name: cluster-api
   namespace: capi-system
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "2"
+    {{- end }}
+    "argocd.argoproj.io/sync-wave": "2"
 {{- with .Values.configSecret }}
 spec:
   configSecret:
@@ -28,4 +34,3 @@ spec:
 {{- end }}
 {{- end }}
 {{- end }}
-

packages/system/capi-operator/charts/cluster-api-operator/templates/core.yaml

@@ -25,8 +25,11 @@ apiVersion: v1
 kind: Namespace
 metadata:
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "1"
+    {{- end }}
+    "argocd.argoproj.io/sync-wave": "1"
   name: {{ $coreNamespace }}
 ---
 apiVersion: operator.cluster.x-k8s.io/v1alpha2
@@ -35,8 +38,10 @@ metadata:
   name: {{ $coreName }}
   namespace: {{ $coreNamespace }}
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "2"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "2"
 {{- if or $coreVersion $.Values.configSecret.name $.Values.manager }}
 spec:
@@ -45,8 +50,8 @@ spec:
   version: {{ $coreVersion }}
 {{- end }}
 {{- if $.Values.manager }}
-  manager:
 {{- if and $.Values.manager.featureGates $.Values.manager.featureGates.core }}
+  manager:
     featureGates:
     {{- range $key, $value := $.Values.manager.featureGates.core }}
       {{ $key }}: {{ $value }}

packages/system/capi-operator/charts/cluster-api-operator/templates/infra-conditions.yaml

@@ -7,8 +7,10 @@ apiVersion: v1
 kind: Namespace
 metadata:
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "1"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "1"
   name: capi-kubeadm-bootstrap-system
 ---
@@ -18,8 +20,10 @@ metadata:
   name: kubeadm
   namespace: capi-kubeadm-bootstrap-system
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "2"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "2"
 {{- with .Values.configSecret }}
 spec:
@@ -37,8 +41,10 @@ apiVersion: v1
 kind: Namespace
 metadata:
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "1"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "1"
   name: capi-kubeadm-control-plane-system
 ---
@@ -48,14 +54,16 @@ metadata:
   name: kubeadm
   namespace: capi-kubeadm-control-plane-system
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "2"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "2"
 {{- with .Values.configSecret }}
 spec:
 {{- if $.Values.manager }}
-  manager:
 {{- if and $.Values.manager.featureGates $.Values.manager.featureGates.kubeadm }}
+  manager:
     featureGates:
     {{- range $key, $value := $.Values.manager.featureGates.kubeadm }}
       {{ $key }}: {{ $value }}

packages/system/capi-operator/charts/cluster-api-operator/templates/infra.yaml

@@ -26,8 +26,10 @@ apiVersion: v1
 kind: Namespace
 metadata:
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "1"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "1"
   name: {{ $infrastructureNamespace }}
 ---
@@ -37,8 +39,10 @@ metadata:
   name: {{ $infrastructureName }}
   namespace: {{ $infrastructureNamespace }}
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "2"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "2"
 {{- if or $infrastructureVersion $.Values.configSecret.name $.Values.manager $.Values.additionalDeployments }}
 spec:
@@ -47,8 +51,8 @@ spec:
   version: {{ $infrastructureVersion }}
 {{- end }}
 {{- if $.Values.manager }}
-  manager:
 {{- if and (kindIs "map" $.Values.manager.featureGates) (hasKey $.Values.manager.featureGates $infrastructureName) }}
+  manager:
 {{- range $key, $value := $.Values.manager.featureGates }}
   {{- if eq $key $infrastructureName }}
     featureGates:

packages/system/capi-operator/charts/cluster-api-operator/templates/ipam.yaml

@@ -26,8 +26,10 @@ apiVersion: v1
 kind: Namespace
 metadata:
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "1"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "1"
   name: {{ $ipamNamespace }}
 ---
@@ -37,8 +39,10 @@ metadata:
   name: {{ $ipamName }}
   namespace: {{ $ipamNamespace }}
   annotations:
+    {{- if $.Values.enableHelmHook }}
     "helm.sh/hook": "post-install,post-upgrade"
     "helm.sh/hook-weight": "2"
+    {{- end }}
     "argocd.argoproj.io/sync-wave": "2"
 {{- if or $ipamVersion $.Values.configSecret.name $.Values.manager $.Values.additionalDeployments }}
 spec:
@@ -47,8 +51,8 @@ spec:
   version: {{ $ipamVersion }}
 {{- end }}
 {{- if $.Values.manager }}
-  manager:
 {{- if and (kindIs "map" $.Values.manager.featureGates) (hasKey $.Values.manager.featureGates $ipamName) }}
+  manager:
 {{- range $key, $value := $.Values.manager.featureGates }}
   {{- if eq $key $ipamName }}
     featureGates:

packages/system/capi-operator/charts/cluster-api-operator/values.yaml

@@ -21,7 +21,7 @@ leaderElection:
 image:
   manager:
     repository: registry.k8s.io/capi-operator/cluster-api-operator
-    tag: v0.17.0
+    tag: v0.18.1
     pullPolicy: IfNotPresent
 env:
   manager: []
@@ -69,3 +69,4 @@ volumeMounts:
     - mountPath: /tmp/k8s-webhook-server/serving-certs
       name: cert
       readOnly: true
+enableHelmHook: true

packages/system/capi-providers/templates/providers.yaml

@@ -21,6 +21,9 @@ spec:
         limits:
           cpu: "1"
           memory: 1024Mi
+        requests:
+          cpu: "10m"
+          memory: 128Mi
 ---
 apiVersion: operator.cluster.x-k8s.io/v1alpha2
 kind: BootstrapProvider

packages/system/cilium-networkpolicy/Chart.yaml

@@ -0,0 +1,3 @@
+apiVersion: v2
+name: cozy-cilium-networkpolicy
+version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process

packages/system/cilium-networkpolicy/Makefile

@@ -0,0 +1,5 @@
+export NAME=cilium-networkpolicy
+export NAMESPACE=cozy-$(NAME)
+
+include ../../../scripts/common-envs.mk
+include ../../../scripts/package.mk