[docs] Add release notes for v0.31.2

Resolves #1060

Signed-off-by: Nick Volynkin <nick.volynkin@gmail.com>

.github/CODEOWNERS

@@ -1 +1 @@
-* @kvaps @lllamnyp
+* @kvaps @lllamnyp @klinch0

.github/workflows/backport.yaml

@@ -0,0 +1,53 @@
+name: Automatic Backport
+
+on:
+  pull_request_target:
+    types: [closed]   # fires when PR is closed (merged)
+
+concurrency:
+  group: backport-${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  backport:
+    if: |
+      github.event.pull_request.merged == true &&
+      contains(github.event.pull_request.labels.*.name, 'backport')
+    runs-on: [self-hosted]
+
+    steps:
+      # 1. Decide which maintenance branch should receive the back‑port
+      - name: Determine target maintenance branch
+        id: target
+        uses: actions/github-script@v7
+        with:
+          script: |
+            let rel;
+            try {
+              rel = await github.rest.repos.getLatestRelease({
+                owner: context.repo.owner,
+                repo: context.repo.repo
+              });
+            } catch (e) {
+              core.setFailed('No existing releases found; cannot determine backport target.');
+              return;
+            }
+            const [maj, min] = rel.data.tag_name.replace(/^v/, '').split('.');
+            const branch = `release-${maj}.${min}`;
+            core.setOutput('branch', branch);
+            console.log(`Latest release ${rel.data.tag_name}; backporting to ${branch}`);
+      # 2. Checkout (required by backport‑action)
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # 3. Create the back‑port pull request
+      - name: Create back‑port PR
+        uses: korthout/backport-action@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          label_pattern: '' # don't read labels for targets
+          target_branches: ${{ steps.target.outputs.branch }}

.github/workflows/pre-commit.yml

@@ -1,6 +1,12 @@
 name: Pre-Commit Checks
 
-on: [push, pull_request]
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+
+concurrency:
+  group: pre-commit-${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
 
 jobs:
   pre-commit:
@@ -8,6 +14,9 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          fetch-tags: true
 
       - name: Set up Python
         uses: actions/setup-python@v4

.github/workflows/pull-requests-release.yaml

@@ -0,0 +1,254 @@
+name: Releasing PR
+
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened, closed]
+
+concurrency:
+  group: pull-requests-release-${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  verify:
+    name: Test Release
+    runs-on: [self-hosted]
+    permissions:
+      contents: read
+      packages: write
+
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'release') &&
+      github.event.action != 'closed'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+
+      - name: Extract tag from PR branch
+        id: get_tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const branch = context.payload.pull_request.head.ref;
+            const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
+            if (!m) {
+              core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);
+              return;
+            }
+            const tag = `v${m[1]}`;
+            core.setOutput('tag', tag);
+
+      - name: Find draft release and get asset IDs
+        id: fetch_assets
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GH_PAT }}
+          script: |
+            const tag = '${{ steps.get_tag.outputs.tag }}';
+            const releases = await github.rest.repos.listReleases({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              per_page: 100
+            });
+            const draft = releases.data.find(r => r.tag_name === tag && r.draft);
+            if (!draft) {
+              core.setFailed(`Draft release '${tag}' not found`);
+              return;
+            }
+            const findAssetId = (name) =>
+              draft.assets.find(a => a.name === name)?.id;
+            const installerId = findAssetId("cozystack-installer.yaml");
+            const diskId = findAssetId("nocloud-amd64.raw.xz");
+            if (!installerId || !diskId) {
+              core.setFailed("Missing required assets");
+              return;
+            }
+            core.setOutput("installer_id", installerId);
+            core.setOutput("disk_id", diskId);
+      
+      - name: Download assets from GitHub API
+        run: |
+          mkdir -p _out/assets
+          curl -sSL \
+            -H "Authorization: token ${GH_PAT}" \
+            -H "Accept: application/octet-stream" \
+            -o _out/assets/cozystack-installer.yaml \
+            "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${{ steps.fetch_assets.outputs.installer_id }}"
+          curl -sSL \
+            -H "Authorization: token ${GH_PAT}" \
+            -H "Accept: application/octet-stream" \
+            -o _out/assets/nocloud-amd64.raw.xz \
+            "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${{ steps.fetch_assets.outputs.disk_id }}"
+        env:
+          GH_PAT: ${{ secrets.GH_PAT }}
+
+      - name: Run tests
+        run: make test
+
+  finalize:
+    name: Finalize Release
+    runs-on: [self-hosted]
+    permissions:
+      contents: write
+
+    if: |
+      github.event.pull_request.merged == true &&
+      contains(github.event.pull_request.labels.*.name, 'release')
+
+    steps:
+      # Extract tag from branch name  (branch = release-X.Y.Z*)
+      - name: Extract tag from branch name
+        id: get_tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const branch = context.payload.pull_request.head.ref;
+            const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
+            if (!m) {
+              core.setFailed(`Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);
+              return;
+            }
+            const tag = `v${m[1]}`;
+            core.setOutput('tag', tag);
+            console.log(`✅ Tag to publish: ${tag}`);
+
+      # Checkout repo & create / push annotated tag
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Create tag on merge commit
+        run: |
+          git tag -f ${{ steps.get_tag.outputs.tag }} ${{ github.sha }}
+          git push -f origin ${{ steps.get_tag.outputs.tag }}
+
+      # Ensure maintenance branch release-X.Y
+      - name: Ensure maintenance branch release-X.Y
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GH_PAT }}
+          script: |
+            const tag = '${{ steps.get_tag.outputs.tag }}';  // e.g. v0.1.3 or v0.1.3-rc3
+            const match = tag.match(/^v(\d+)\.(\d+)\.\d+(?:[-\w\.]+)?$/);
+            if (!match) {
+              core.setFailed(`❌ tag '${tag}' must match 'vX.Y.Z' or 'vX.Y.Z-suffix'`);
+              return;
+            }
+            const line = `${match[1]}.${match[2]}`;
+            const branch = `release-${line}`;
+
+            // Get main branch commit for the tag
+            const ref = await github.rest.git.getRef({
+              owner: context.repo.owner,
+              repo:  context.repo.repo,
+              ref:   `tags/${tag}`
+            });
+
+            const commitSha = ref.data.object.sha;
+
+            try {
+              await github.rest.repos.getBranch({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                branch
+              });
+           
+              await github.rest.git.updateRef({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                ref: `heads/${branch}`,
+                sha: commitSha,
+                force: true
+              });
+              console.log(`🔁 Force-updated '${branch}' to ${commitSha}`);
+            } catch (err) {
+              if (err.status === 404) {
+                await github.rest.git.createRef({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  ref: `refs/heads/${branch}`,
+                  sha: commitSha
+                });
+                console.log(`✅ Created branch '${branch}' at ${commitSha}`);
+              } else {
+                console.error('Unexpected error --', err);
+                core.setFailed(`Unexpected error creating/updating branch: ${err.message}`);
+                throw err;
+              }
+            }
+
+      # Get the latest published release
+      - name: Get the latest published release
+        id: latest_release
+        uses: actions/github-script@v7
+        with:
+          script: |
+            try {
+              const rel = await github.rest.repos.getLatestRelease({
+                owner: context.repo.owner,
+                repo:  context.repo.repo
+              });
+              core.setOutput('tag', rel.data.tag_name);
+            } catch (_) {
+              core.setOutput('tag', '');
+            }
+
+      # Compare current tag vs latest using semver-utils
+      - name: Semver compare
+        id: semver
+        uses: madhead/semver-utils@v4.3.0
+        with:
+          version:    ${{ steps.get_tag.outputs.tag }}
+          compare-to: ${{ steps.latest_release.outputs.tag }}
+
+      # Derive flags: prerelease?  make_latest?
+      - name: Calculate publish flags
+        id: flags
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag = '${{ steps.get_tag.outputs.tag }}';              // v0.31.5-rc.1
+            const m = tag.match(/^v(\d+\.\d+\.\d+)(-(?:alpha|beta|rc)\.\d+)?$/);
+            if (!m) {
+              core.setFailed(`❌ tag '${tag}' must match 'vX.Y.Z' or 'vX.Y.Z-(alpha|beta|rc).N'`);
+              return;
+            }
+            const version = m[1] + (m[2] ?? '');                         // 0.31.5-rc.1
+            const isRc    = Boolean(m[2]);
+            core.setOutput('is_rc',      isRc);
+            const outdated = '${{ steps.semver.outputs.comparison-result }}' === '<';
+            core.setOutput('make_latest', isRc || outdated ? 'false' : 'legacy');
+
+      # Publish draft release with correct flags
+      - name: Publish draft release
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag = '${{ steps.get_tag.outputs.tag }}';
+            const releases = await github.rest.repos.listReleases({
+              owner: context.repo.owner,
+              repo:  context.repo.repo
+            });
+            const draft = releases.data.find(r => r.tag_name === tag && r.draft);
+            if (!draft) throw new Error(`Draft release for ${tag} not found`);
+            await github.rest.repos.updateRelease({
+              owner:       context.repo.owner,
+              repo:        context.repo.repo,
+              release_id:  draft.id,
+              draft:       false,
+              prerelease:  ${{ steps.flags.outputs.is_rc }},
+              make_latest: '${{ steps.flags.outputs.make_latest }}'
+            });
+
+            console.log(`🚀 Published release for ${tag}`);

.github/workflows/pull-requests.yaml

@@ -0,0 +1,82 @@
+name: Pull Request
+
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+
+concurrency:
+  group: pull-requests-${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build
+    runs-on: [self-hosted]
+    permissions:
+      contents: read
+      packages: write
+
+    # Never run when the PR carries the "release" label.
+    if: |
+      !contains(github.event.pull_request.labels.*.name, 'release')
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+        env:
+          DOCKER_CONFIG: ${{ runner.temp }}/.docker
+
+      - name: Build
+        run: make build
+        env:
+          DOCKER_CONFIG: ${{ runner.temp }}/.docker
+
+      - name: Build Talos image
+        run: make -C packages/core/installer talos-nocloud
+     
+      - name: Upload installer
+        uses: actions/upload-artifact@v4
+        with:
+          name: cozystack-installer
+          path: _out/assets/cozystack-installer.yaml
+
+      - name: Upload Talos image
+        uses: actions/upload-artifact@v4
+        with:
+          name: talos-image
+          path: _out/assets/nocloud-amd64.raw.xz
+ 
+  test:
+    name: Test
+    runs-on: [self-hosted]
+    needs: build
+
+    # Never run when the PR carries the "release" label.
+    if: |
+      !contains(github.event.pull_request.labels.*.name, 'release')
+
+    steps:
+      - name: Download installer
+        uses: actions/download-artifact@v4
+        with:
+          name: cozystack-installer
+          path: _out/assets/
+
+      - name: Download Talos image
+        uses: actions/download-artifact@v4
+        with:
+          name: talos-image
+          path: _out/assets/
+
+      - name: Test
+        run: make test

.github/workflows/tags.yaml

@@ -0,0 +1,232 @@
+name: Versioned Tag
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'          # vX.Y.Z
+      - 'v*.*.*-rc.*'     # vX.Y.Z-rc.N
+      - 'v*.*.*-beta.*'   # vX.Y.Z-beta.N
+      - 'v*.*.*-alpha.*'  # vX.Y.Z-alpha.N
+
+concurrency:
+  group: tags-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  prepare-release:
+    name: Prepare Release
+    runs-on: [self-hosted]
+    permissions:
+      contents: write
+      packages: write
+      pull-requests: write
+      actions: write
+
+    steps:
+      # Check if a non-draft release with this tag already exists
+      - name: Check if release already exists
+        id: check_release
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag = context.ref.replace('refs/tags/', '');
+            const releases = await github.rest.repos.listReleases({
+              owner: context.repo.owner,
+              repo:  context.repo.repo
+            });
+            const exists = releases.data.some(r => r.tag_name === tag && !r.draft);
+            core.setOutput('skip', exists);
+            console.log(exists ? `Release ${tag} already published` : `No published release ${tag}`);
+
+      # If a published release already exists, skip the rest of the workflow
+      - name: Skip if release already exists
+        if: steps.check_release.outputs.skip == 'true'
+        run: echo "Release already exists, skipping workflow."
+
+      # Parse tag meta-data (rc?, maintenance line, etc.)
+      - name: Parse tag
+        if: steps.check_release.outputs.skip == 'false'
+        id: tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const ref = context.ref.replace('refs/tags/', '');           // e.g. v0.31.5-rc.1
+            const m = ref.match(/^v(\d+\.\d+\.\d+)(-(?:alpha|beta|rc)\.\d+)?$/);        // ['0.31.5', '-rc.1' | '-beta.1' | …]
+            if (!m) {
+              core.setFailed(`❌ tag '${ref}' must match 'vX.Y.Z' or 'vX.Y.Z-(alpha|beta|rc).N'`);
+              return;
+            }
+            const version = m[1] + (m[2] ?? '');                         // 0.31.5-rc.1
+            const isRc    = Boolean(m[2]);
+            const [maj, min] = m[1].split('.');
+            core.setOutput('tag',     ref);                              // v0.31.5-rc.1
+            core.setOutput('version', version);                          // 0.31.5-rc.1
+            core.setOutput('is_rc',   isRc);                             // true
+            core.setOutput('line',    `${maj}.${min}`);                  // 0.31
+
+      # Detect base branch (main or release-X.Y) the tag was pushed from
+      - name: Get base branch
+        if: steps.check_release.outputs.skip == 'false'
+        id: get_base
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const baseRef = context.payload.base_ref;
+            if (!baseRef) {
+              core.setFailed(`❌ base_ref is empty. Push the tag via 'git push origin HEAD:refs/tags/<tag>'.`);
+              return;
+            }
+            const branch = baseRef.replace('refs/heads/', '');
+            const ok     = branch === 'main' || /^release-\d+\.\d+$/.test(branch);
+            if (!ok) {
+              core.setFailed(`❌ Tagged commit must belong to 'main' or 'release-X.Y'. Got '${branch}'`);
+              return;
+            }
+            core.setOutput('branch', branch);
+
+      # Checkout & login once
+      - name: Checkout code
+        if: steps.check_release.outputs.skip == 'false'
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags:  true
+
+      - name: Login to GHCR
+        if: steps.check_release.outputs.skip == 'false'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+        env:
+          DOCKER_CONFIG: ${{ runner.temp }}/.docker
+
+      # Build project artifacts
+      - name: Build
+        if: steps.check_release.outputs.skip == 'false'
+        run: make build
+        env:
+          DOCKER_CONFIG: ${{ runner.temp }}/.docker
+
+      # Commit built artifacts
+      - name: Commit release artifacts
+        if: steps.check_release.outputs.skip == 'false'
+        run: |
+          git config user.name  "github-actions"
+          git config user.email "github-actions@github.com"
+          git add .
+          git commit -m "Prepare release ${GITHUB_REF#refs/tags/}" -s || echo "No changes to commit"
+          git push origin HEAD || true
+
+      # Get `latest_version` from latest published release 
+      - name: Get latest published release
+        if: steps.check_release.outputs.skip == 'false'
+        id: latest_release
+        uses: actions/github-script@v7
+        with:
+          script: |
+            try {
+              const rel = await github.rest.repos.getLatestRelease({
+                owner: context.repo.owner,
+                repo:  context.repo.repo
+              });
+              core.setOutput('tag', rel.data.tag_name);
+            } catch (_) {
+              core.setOutput('tag', '');
+            }
+
+      # Compare tag (A) with latest (B)
+      - name: Semver compare
+        if: steps.check_release.outputs.skip == 'false'
+        id: semver
+        uses: madhead/semver-utils@v4.3.0
+        with:
+          version:     ${{ steps.tag.outputs.tag }}            # A
+          compare-to:  ${{ steps.latest_release.outputs.tag }} # B
+
+      # Create or reuse DRAFT GitHub Release
+      - name: Create / reuse draft release
+        if: steps.check_release.outputs.skip == 'false'
+        id: release
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag        = '${{ steps.tag.outputs.tag }}';
+            const isRc       = ${{ steps.tag.outputs.is_rc }};
+            const outdated   = '${{ steps.semver.outputs.comparison-result }}' === '<';
+            const makeLatest = outdated ? false : 'legacy';
+            const releases   = await github.rest.repos.listReleases({
+              owner: context.repo.owner,
+              repo:  context.repo.repo
+            });
+            let rel          = releases.data.find(r => r.tag_name === tag);
+            if (!rel) {
+              rel = await github.rest.repos.createRelease({
+                owner: context.repo.owner,
+                repo:  context.repo.repo,
+                tag_name:    tag,
+                name:        tag,
+                draft:       true,
+                prerelease:  isRc,
+                make_latest: makeLatest
+              });
+              console.log(`Draft release created for ${tag}`);
+            } else {
+              console.log(`Re-using existing release ${tag}`);
+            }
+            core.setOutput('upload_url', rel.upload_url);
+
+      # Build + upload assets (optional)
+      - name: Build & upload assets
+        if: steps.check_release.outputs.skip == 'false'
+        run: |
+          make assets
+          make upload_assets VERSION=${{ steps.tag.outputs.tag }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # Create release-X.Y.Z branch and push (force-update)
+      - name: Create release branch
+        if: steps.check_release.outputs.skip == 'false'
+        run: |
+          BRANCH="release-${GITHUB_REF#refs/tags/v}"
+          git branch -f "$BRANCH"
+          git push -f origin "$BRANCH"
+
+      # Create pull request into original base branch (if absent)
+      - name: Create pull request if not exists
+        if: steps.check_release.outputs.skip == 'false'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const version = context.ref.replace('refs/tags/v', '');
+            const base    = '${{ steps.get_base.outputs.branch }}';
+            const head    = `release-${version}`;
+
+            const prs = await github.rest.pulls.list({
+              owner: context.repo.owner,
+              repo:  context.repo.repo,
+              head:  `${context.repo.owner}:${head}`,
+              base
+            });
+            if (prs.data.length === 0) {
+              const pr = await github.rest.pulls.create({
+                owner: context.repo.owner,
+                repo:  context.repo.repo,
+                head,
+                base,
+                title: `Release v${version}`,
+                body:  `This PR prepares the release \`v${version}\`.`,
+                draft: false
+              });
+              await github.rest.issues.addLabels({
+                owner: context.repo.owner,
+                repo:  context.repo.repo,
+                issue_number: pr.data.number,
+                labels: ['release']
+              });
+              console.log(`Created PR #${pr.data.number}`);
+            } else {
+              console.log(`PR already exists from ${head} to ${base}`);
+            }

.gitignore

@@ -1,6 +1,7 @@
 _out
 .git
 .idea
+.vscode
 
 # User-specific stuff
 .idea/**/workspace.xml
@@ -75,4 +76,4 @@ fabric.properties
 .idea/caches/build_file_checksums.ser
 
 .DS_Store
-**/.DS_Store
+**/.DS_Store

.pre-commit-config.yaml

@@ -18,6 +18,7 @@ repos:
               (cd "$dir" && make generate)
             fi
           done
+          git diff --color=always | cat
         '
       language: script
       files: ^.*$

CONTRIBUTING.md

@@ -6,13 +6,13 @@ As you get started, you are in the best position to give us feedbacks on areas o
 
 * Problems found while setting up the development environment
 * Gaps in our documentation
-* Bugs in our Github actions
+* Bugs in our GitHub actions
 
-First, though, it is important that you read the [code of conduct](CODE_OF_CONDUCT.md).
+First, though, it is important that you read the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
 
 The guidelines below are a starting point. We don't want to limit your
 creativity, passion, and initiative. If you think there's a better way, please
-feel free to bring it up in a Github discussion, or open a pull request. We're
+feel free to bring it up in a GitHub discussion, or open a pull request. We're
 certain there are always better ways to do things, we just need to start some
 constructive dialogue!
 
@@ -23,9 +23,9 @@ We welcome many types of contributions including:
 * New features
 * Builds, CI/CD
 * Bug fixes
-* [Documentation](https://github.com/cozystack/cozystack-website/tree/main)
+* [Documentation](https://GitHub.com/cozystack/cozystack-website/tree/main)
 * Issue Triage
-* Answering questions on Slack or Github Discussions
+* Answering questions on Slack or GitHub Discussions
 * Web design
 * Communications / Social Media / Blog Posts
 * Events participation
@@ -34,7 +34,7 @@ We welcome many types of contributions including:
 ## Ask for Help
 
 The best way to reach us with a question when contributing is to drop a line in
-our [Telegram channel](https://t.me/cozystack), or start a new Github discussion.
+our [Telegram channel](https://t.me/cozystack), or start a new GitHub discussion.
 
 ## Raising Issues
 

GOVERNANCE.md

@@ -0,0 +1,91 @@
+# Cozystack Governance
+
+This document defines the governance structure of the Cozystack community, outlining how members collaborate to achieve shared goals.
+
+## Overview
+
+**Cozystack**, a Cloud Native Computing Foundation (CNCF) project, is committed
+to building an open, inclusive, productive, and self-governing open source
+community focused on building a high-quality open source PaaS and framework for building clouds.
+
+## Code Repositories
+
+The following code repositories are governed by the Cozystack community and
+maintained under the `cozystack` namespace:
+
+* **[Cozystack](https://github.com/cozystack/cozystack):** Main Cozystack codebase
+* **[website](https://github.com/cozystack/website):** Cozystack website and documentation sources
+* **[Talm](https://github.com/cozystack/talm):** Tool for managing Talos Linux the GitOps way
+* **[cozy-proxy](https://github.com/cozystack/cozy-proxy):** A simple kube-proxy addon for 1:1 NAT services in Kubernetes with NFT backend
+* **[cozystack-telemetry-server](https://github.com/cozystack/cozystack-telemetry-server):** Cozystack telemetry
+* **[talos-bootstrap](https://github.com/cozystack/talos-bootstrap):** An interactive Talos Linux installer
+* **[talos-meta-tool](https://github.com/cozystack/talos-meta-tool):** Tool for writing network metadata into META partition
+
+## Community Roles
+
+* **Users:** Members that engage with the Cozystack community via any medium, including Slack, Telegram, GitHub, and mailing lists.
+* **Contributors:** Members contributing to the projects by contributing and reviewing code, writing documentation,
+  responding to issues, participating in proposal discussions, and so on.
+* **Directors:** Non-technical project leaders.
+* **Maintainers**: Technical project leaders.
+
+## Contributors
+
+Cozystack is for everyone. Anyone can become a Cozystack contributor simply by
+contributing to the project, whether through code, documentation, blog posts,
+community management, or other means.
+As with all Cozystack community members, contributors are expected to follow the
+[Cozystack Code of Conduct](https://github.com/cozystack/cozystack/blob/main/CODE_OF_CONDUCT.md).
+
+All contributions to Cozystack code, documentation, or other components in the
+Cozystack GitHub organisation must follow the 
+[contributing guidelines](https://github.com/cozystack/cozystack/blob/main/CONTRIBUTING.md).
+Whether these contributions are merged into the project is the prerogative of the maintainers.
+
+## Directors
+
+Directors are responsible for non-technical leadership functions within the project.
+This includes representing Cozystack and its maintainers to the community, to the press, 
+and to the outside world; interfacing with CNCF and other governance entities;
+and participating in project decision-making processes when appropriate.
+
+Directors are elected by a majority vote of the maintainers.
+
+## Maintainers
+
+Maintainers have the right to merge code into the project.
+Anyone can become a Cozystack maintainer (see "Becoming a maintainer" below).
+
+### Expectations
+
+Cozystack maintainers are expected to:
+
+* Review pull requests, triage issues, and fix bugs in their areas of
+  expertise, ensuring that all changes go through the project's code review
+  and integration processes.
+* Monitor cncf-cozystack-* emails, the Cozystack Slack channels in Kubernetes
+  and CNCF Slack workspaces, Telegram groups, and help out when possible.
+* Rapidly respond to any time-sensitive security release processes.
+* Attend Cozystack community meetings.
+
+If a maintainer is no longer interested in or cannot perform the duties
+listed above, they should move themselves to emeritus status.
+If necessary, this can also occur through the decision-making process outlined below.
+
+### Becoming a Maintainer
+
+Anyone can become a Cozystack maintainer. Maintainers should be extremely
+proficient in cloud native technologies and/or Go; have relevant domain expertise; 
+have the time and ability to meet the maintainer's expectations above; 
+and demonstrate the ability to work with the existing maintainers and project processes.
+
+To become a maintainer, start by expressing interest to existing maintainers.
+Existing maintainers will then ask you to demonstrate the qualifications above
+by contributing PRs, doing code reviews, and other such tasks under their guidance.
+After several months of working together, maintainers will decide whether to grant maintainer status.
+
+## Project Decision-making Process
+
+Ideally, all project decisions are resolved by consensus of maintainers and directors.
+If this is not possible, a vote will be called.
+The voting process is a simple majority in which each maintainer and director receives one vote.

Makefile

@@ -1,6 +1,13 @@
 .PHONY: manifests repos assets
 
-build:
+build-deps:
+	@command -V find docker skopeo jq gh helm > /dev/null
+	@yq --version | grep -q "mikefarah" || (echo "mikefarah/yq is required" && exit 1)
+	@tar --version | grep -q GNU || (echo "GNU tar is required" && exit 1)
+	@sed --version | grep -q GNU || (echo "GNU sed is required" && exit 1)
+	@awk --version | grep -q GNU || (echo "GNU awk is required" && exit 1)
+
+build: build-deps
 	make -C packages/apps/http-cache image
 	make -C packages/apps/postgres image
 	make -C packages/apps/mysql image
@@ -13,16 +20,13 @@ build:
 	make -C packages/system/kubeovn image
 	make -C packages/system/kubeovn-webhook image
 	make -C packages/system/dashboard image
+	make -C packages/system/metallb image
 	make -C packages/system/kamaji image
 	make -C packages/system/bucket image
 	make -C packages/core/testing image
 	make -C packages/core/installer image
 	make manifests
 
-manifests:
-	(cd packages/core/installer/; helm template -n cozy-installer installer .) > manifests/cozystack-installer.yaml
-	sed -i 's|@sha256:[^"]\+||' manifests/cozystack-installer.yaml
-
 repos:
 	rm -rf _out
 	make -C packages/apps check-version-map
@@ -33,14 +37,20 @@ repos:
 	mkdir -p _out/logos
 	cp ./packages/apps/*/logos/*.svg ./packages/extra/*/logos/*.svg _out/logos/
 
+
+manifests:
+	mkdir -p _out/assets
+	(cd packages/core/installer/; helm template -n cozy-installer installer .) > _out/assets/cozystack-installer.yaml
+
 assets:
-	make -C packages/core/installer/ assets
+	make -C packages/core/installer assets
 
 test:
-	test -f _out/assets/nocloud-amd64.raw.xz || make -C packages/core/installer talos-nocloud
 	make -C packages/core/testing apply
 	make -C packages/core/testing test
-	make -C packages/core/testing test-applications
 
 generate:
 	hack/update-codegen.sh
+
+upload_assets: manifests
+	hack/upload-assets.sh

README.md

@@ -12,20 +12,21 @@
 
 **Cozystack** is a free PaaS platform and framework for building clouds.
 
-With Cozystack, you can transform your bunch of servers into an intelligent system with a simple REST API for spawning Kubernetes clusters, Database-as-a-Service, virtual machines, load balancers, HTTP caching services, and other services with ease.
+With Cozystack, you can transform a bunch of servers into an intelligent system with a simple REST API for spawning Kubernetes clusters,
+Database-as-a-Service, virtual machines, load balancers, HTTP caching services, and other services with ease.
 
-You can use Cozystack to build your own cloud or to provide a cost-effective development environments.  
+Use Cozystack to build your own cloud or provide a cost-effective development environment.  
 
 ## Use-Cases
 
-* [**Using Cozystack to build public cloud**](https://cozystack.io/docs/use-cases/public-cloud/)  
-You can use Cozystack as backend for a public cloud
+* [**Using Cozystack to build a public cloud**](https://cozystack.io/docs/guides/use-cases/public-cloud/)  
+You can use Cozystack as a backend for a public cloud
 
-* [**Using Cozystack to build private cloud**](https://cozystack.io/docs/use-cases/private-cloud/)  
-You can use Cozystack as platform to build a private cloud powered by Infrastructure-as-Code approach
+* [**Using Cozystack to build a private cloud**](https://cozystack.io/docs/guides/use-cases/private-cloud/)  
+You can use Cozystack as a platform to build a private cloud powered by Infrastructure-as-Code approach
 
-* [**Using Cozystack as Kubernetes distribution**](https://cozystack.io/docs/use-cases/kubernetes-distribution/)  
-You can use Cozystack as Kubernetes distribution for Bare Metal
+* [**Using Cozystack as a Kubernetes distribution**](https://cozystack.io/docs/guides/use-cases/kubernetes-distribution/)  
+You can use Cozystack as a Kubernetes distribution for Bare Metal
 
 ## Screenshot
 
@@ -33,11 +34,11 @@ You can use Cozystack as Kubernetes distribution for Bare Metal
 
 ## Documentation
 
-The documentation is located on official [cozystack.io](https://cozystack.io) website.
+The documentation is located on the [cozystack.io](https://cozystack.io) website.
 
-Read [Get Started](https://cozystack.io/docs/get-started/) section for a quick start.
+Read the [Getting Started](https://cozystack.io/docs/getting-started/) section for a quick start.
 
-If you encounter any difficulties, start with the [troubleshooting guide](https://cozystack.io/docs/troubleshooting/), and work your way through the process that we've outlined.
+If you encounter any difficulties, start with the [troubleshooting guide](https://cozystack.io/docs/operations/troubleshooting/) and work your way through the process that we've outlined.
 
 ## Versioning
 
@@ -50,15 +51,15 @@ A full list of the available releases is available in the GitHub repository's [R
 
 Contributions are highly appreciated and very welcomed!
 
-In case of bugs, please, check if the issue has been already opened by checking the [GitHub Issues](https://github.com/cozystack/cozystack/issues) section.
-In case it isn't, you can open a new one: a detailed report will help us to replicate it, assess it, and work on a fix.
+In case of bugs, please check if the issue has already been opened by checking the [GitHub Issues](https://github.com/cozystack/cozystack/issues) section.
+If it isn't, you can open a new one. A detailed report will help us replicate it, assess it, and work on a fix.
 
-You can express your intention in working on the fix on your own.
+You can express your intention to on the fix on your own.
 Commits are used to generate the changelog, and their author will be referenced in it.
 
-In case of **Feature Requests** please use the [Discussion's Feature Request section](https://github.com/cozystack/cozystack/discussions/categories/feature-requests).
+If you have **Feature Requests** please use the [Discussion's Feature Request section](https://github.com/cozystack/cozystack/discussions/categories/feature-requests).
 
-You can join our weekly community meetings (just add this events to your [Google Calendar](https://calendar.google.com/calendar?cid=ZTQzZDIxZTVjOWI0NWE5NWYyOGM1ZDY0OWMyY2IxZTFmNDMzZTJlNjUzYjU2ZGJiZGE3NGNhMzA2ZjBkMGY2OEBncm91cC5jYWxlbmRhci5nb29nbGUuY29t) or [iCal](https://calendar.google.com/calendar/ical/e43d21e5c9b45a95f28c5d649c2cb1e1f433e2e653b56dbbda74ca306f0d0f68%40group.calendar.google.com/public/basic.ics)) or [Telegram group](https://t.me/cozystack).
+You are welcome to join our weekly community meetings (just add this events to your [Google Calendar](https://calendar.google.com/calendar?cid=ZTQzZDIxZTVjOWI0NWE5NWYyOGM1ZDY0OWMyY2IxZTFmNDMzZTJlNjUzYjU2ZGJiZGE3NGNhMzA2ZjBkMGY2OEBncm91cC5jYWxlbmRhci5nb29nbGUuY29t) or [iCal](https://calendar.google.com/calendar/ical/e43d21e5c9b45a95f28c5d649c2cb1e1f433e2e653b56dbbda74ca306f0d0f68%40group.calendar.google.com/public/basic.ics)) or [Telegram group](https://t.me/cozystack).
 
 ## License
 

cmd/cozystack-controller/main.go

@@ -39,6 +39,8 @@ import (
 	cozystackiov1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
 	"github.com/cozystack/cozystack/internal/controller"
 	"github.com/cozystack/cozystack/internal/telemetry"
+
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -51,6 +53,7 @@ func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 
 	utilruntime.Must(cozystackiov1alpha1.AddToScheme(scheme))
+	utilruntime.Must(helmv2.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
 
@@ -178,6 +181,31 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "WorkloadMonitor")
 		os.Exit(1)
 	}
+
+	if err = (&controller.WorkloadReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "WorkloadReconciler")
+		os.Exit(1)
+	}
+
+	if err = (&controller.TenantHelmReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "TenantHelmReconciler")
+		os.Exit(1)
+	}
+
+	if err = (&controller.CozystackConfigReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "CozystackConfigReconciler")
+		os.Exit(1)
+	}
+
 	// +kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

dashboards/flux/flux-control-plane.json

@@ -626,7 +626,7 @@
             "datasource": {
               "uid": "${DS_PROMETHEUS}"
             },
-            "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"POD\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)",
+            "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)",
             "hide": false,
             "interval": "",
             "legendFormat": "{{pod}}",

dashboards/main/capacity-planning.json

@@ -450,7 +450,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval])))\n / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))",
+          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"\",node=~\"$node\"}[$__rate_interval])))\n / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -520,7 +520,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"})) / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",unit=\"byte\",node=~\"$node\"}[$__rate_interval])))",
+          "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"\",node=~\"$node\"})) / sum(sum by (node) (avg_over_time(kube_node_status_allocatable{resource=\"memory\",unit=\"byte\",node=~\"$node\"}[$__rate_interval])))",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -590,7 +590,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval]))) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))",
+          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"\",node=~\"$node\"}[$__rate_interval]))) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\"}[$__rate_interval])))",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -660,7 +660,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"} )) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"}[$__rate_interval])))",
+          "expr": "sum(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"\",node=~\"$node\"} )) / sum(sum by (node) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"}[$__rate_interval])))",
           "hide": false,
           "legendFormat": "__auto",
           "range": true,
@@ -1128,7 +1128,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0\n",
+          "expr": "sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0\n",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -1143,7 +1143,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0)",
+          "expr": "sum(sum by (node) (rate(container_cpu_usage_seconds_total{container!=\"\",node=~\"$node\"}[$__rate_interval]) - on (namespace,pod,container,node) group_left avg by (namespace,pod,container, node)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"})) * -1 > 0)",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -1527,7 +1527,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0\n",
+          "expr": "(sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0\n",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -1542,7 +1542,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum((sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0)",
+          "expr": "sum((sum by (node) (container_memory_working_set_bytes:without_kmem{container!=\"\",node=~\"$node\"} ) - sum by (node) (kube_pod_container_resource_requests{resource=\"memory\",node=~\"$node\"})) * -1 > 0)",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -1909,7 +1909,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) * -1 > 0)\n",
+          "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) * -1 > 0)\n",
           "format": "table",
           "instant": true,
           "range": false,
@@ -2037,7 +2037,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) * -1 >0)\n",
+          "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) * -1 >0)\n",
           "format": "table",
           "instant": true,
           "range": false,
@@ -2160,7 +2160,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"POD\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) > 0)\n",
+          "expr": "topk(10, (sum by (namespace,pod,container)((rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"\",node=~\"$node\"}[$__rate_interval])) - on (namespace,pod,container) group_left avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"cpu\",node=~\"$node\"}))) > 0)\n",
           "format": "table",
           "instant": true,
           "range": false,
@@ -2288,7 +2288,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"POD\",container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) >0)\n",
+          "expr": "topk(10, (sum by (namespace,container,pod) (container_memory_working_set_bytes:without_kmem{container!=\"\",namespace=~\"$namespace\",node=~\"$node\"}) - on (namespace,pod,container) avg by (namespace,pod,container)(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\",node=~\"$node\"})) >0)\n",
           "format": "table",
           "instant": true,
           "range": false,

dashboards/main/controller.json

@@ -684,7 +684,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])  * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])  * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -710,7 +710,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (pod)\n(\n  avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\", namespace=\"$namespace\", resource=\"cpu\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "sum by (pod)\n(\n  avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\", namespace=\"$namespace\", resource=\"cpu\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -723,7 +723,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n    -\n    sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n  ) > 0\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n    -\n    sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n  ) > 0\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -736,7 +736,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) \n  (\n    (\n      (\n        sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n      ) or sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n    ) > 0\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) \n  (\n    (\n      (\n        sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n      ) or sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n    ) > 0\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -762,7 +762,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -786,7 +786,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (pod)\n(\n  avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\", namespace=\"$namespace\", resource=\"memory\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "sum by (pod)\n(\n  avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\", namespace=\"$namespace\", resource=\"memory\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -798,7 +798,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    (\n      (\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n      ) > 0\n    )\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    (\n      (\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n      ) > 0\n    )\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -810,7 +810,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    (\n      (\n        sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n      ) or sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__range]))\n    ) > 0\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod)\n  (\n    (\n      (\n        sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n        -\n        sum by (namespace, pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n      ) or sum by (namespace, pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__range]))\n    ) > 0\n  )\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -848,7 +848,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -860,7 +860,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
+          "expr": "(\n  sum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range])) \n  * on (pod)\n  sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__range]))\n)\nor\nsum by (pod) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -1315,7 +1315,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -1488,7 +1488,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -1502,7 +1502,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -1642,7 +1642,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by (pod)\n  (\n    max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n    * on (pod)\n    sum by (pod) (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (pod)\n  (\n    max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n    * on (pod)\n    sum by (pod) (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -1779,7 +1779,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "    (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (\n    (\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n    )\n    or\n    sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n  )\n) > 0",
+          "expr": "    (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (\n    (\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n    )\n    or\n    sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))\n  )\n) > 0",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -2095,7 +2095,7 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Usage",
@@ -2109,7 +2109,7 @@
               "refId": "D"
             },
             {
-              "expr": "sum by (pod)\n(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\", namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))\n)",
+              "expr": "sum by (pod)\n(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\", namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))\n)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
@@ -2295,7 +2295,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by(pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "System",
@@ -2306,7 +2306,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by(pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "User",
@@ -2468,7 +2468,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -2653,7 +2653,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "RSS",
@@ -2666,7 +2666,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Cache",
@@ -2679,7 +2679,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Swap",
@@ -2692,7 +2692,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Working set bytes without kmem",
@@ -2705,7 +2705,7 @@
             "uid": "${ds_prometheus}"
           },
           "editorMode": "code",
-          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))\n)",
+          "expr": "sum (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}) by(pod)\n  * on (pod)\n  sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Kmem",
@@ -2837,7 +2837,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (pod) group_left()\n  sum by (pod)\n    (\n      (\n        sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n        -\n        sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n      ) > 0\n    )\n)",
+          "expr": "(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (pod) group_left()\n  sum by (pod)\n    (\n      (\n        sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n        -\n        sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n      ) > 0\n    )\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -2974,7 +2974,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (pod) group_left()\n  sum by (pod)\n    (\n      (\n        (\n          sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n        ) or sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n      ) > 0\n    )\n)",
+          "expr": "(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (pod) group_left()\n  sum by (pod)\n    (\n      (\n        (\n          sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n        ) or sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n      ) > 0\n    )\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -3290,56 +3290,56 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "RSS",
               "refId": "A"
             },
             {
-              "expr": "sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Cache",
               "refId": "B"
             },
             {
-              "expr": "sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Swap",
               "refId": "C"
             },
             {
-              "expr": "sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Working set bytes without kmem",
               "refId": "D"
             },
             {
-              "expr": "sum by (pod)\n(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", container!=\"POD\", resource=\"memory\"}[$__rate_interval]))\n)",
+              "expr": "sum by (pod)\n(\n  kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\", pod=~\"$pod\"}\n  * on (controller_type, controller_name) group_left()\n  sum by (controller_type, controller_name) (avg_over_time(vpa_target_recommendation{namespace=\"$namespace\", container!=\"\", resource=\"memory\"}[$__rate_interval]))\n)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
               "refId": "E"
             },
             {
-              "expr": "sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "F"
             },
             {
-              "expr": "sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "G"
             },
             {
-              "expr": "sum by(pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by(pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", pod=~\"$pod\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Kmem",
@@ -3834,7 +3834,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",
@@ -3972,7 +3972,7 @@
             "uid": "$ds_prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"POD\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
+          "expr": "sum by(pod) (\n  max(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}) by(pod)\n  * on (pod)\n  sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"\", pod=~\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ pod }}",

dashboards/main/namespace.json

@@ -656,7 +656,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -680,7 +680,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"cpu\"}[$__range]))\n  ) \nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"cpu\"}[$__range]))\n  ) \nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -692,7 +692,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n          -\n          sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n          -\n          sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -704,7 +704,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n          ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n          ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -728,7 +728,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -740,7 +740,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (pod) group_left()\n    sum by (namespace, pod)\n      (\n        avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__range])\n      )\n    )\n  or\n  count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (pod) group_left()\n    sum by (namespace, pod)\n      (\n        avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__range])\n      )\n    )\n  or\n  count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -752,7 +752,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"memory\"}[$__range]))\n  ) \n  or  \ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"memory\"}[$__range]))\n  ) \n  or  \ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -764,7 +764,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n          -\n          sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n          -\n          sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -776,7 +776,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n          ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller)\n  (\n    avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__range]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n          ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range]))\n        ) > 0\n      )\n  )\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -814,7 +814,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -826,7 +826,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -877,7 +877,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
+          "expr": "sum by (controller) (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range]) * on (pod) group_left() sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}[$__range])) by (controller) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -1475,7 +1475,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval])))",
+          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval])))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -1646,7 +1646,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum (sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))))",
+          "expr": "sum (sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "System",
@@ -1657,7 +1657,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum (sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))))",
+          "expr": "sum (sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "User",
@@ -1798,7 +1798,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -1939,7 +1939,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n          ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n          ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -2257,28 +2257,28 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval])))",
+              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval])))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Usage",
               "refId": "D"
             },
             {
-              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
+              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "C"
             },
             {
-              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
+              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "E"
             },
             {
-              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))\n  )",
+              "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
@@ -2458,7 +2458,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval])))",
+              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval])))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -2470,7 +2470,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval])))",
+              "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval])))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "User",
@@ -2622,7 +2622,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -2799,14 +2799,14 @@
       "pluginVersion": "8.5.13",
       "targets": [
         {
-          "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "RSS",
           "refId": "A"
         },
         {
-          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -2814,7 +2814,7 @@
           "refId": "B"
         },
         {
-          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -2822,14 +2822,14 @@
           "refId": "C"
         },
         {
-          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Working set bytes without kmem",
           "refId": "D"
         },
         {
-          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+          "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Kmem",
@@ -2955,7 +2955,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n          -\n          sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n        ) > 0\n      )\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -3091,7 +3091,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n          )\n          or\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n            +\n            sum by(namespace, pod, container) (avg_over_time(container_memory:kmem{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))\n          )\n        ) > 0\n      )\n  )",
+          "expr": "sum by (controller)\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"}\n    * on (namespace, pod) group_left()\n    sum by (namespace, pod)\n      (\n        (\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\"}[$__rate_interval]))\n            -\n            sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n          )\n          or\n          (\n            sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n            +\n            sum by(namespace, pod, container) (avg_over_time(container_memory:kmem{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))\n          )\n        ) > 0\n      )\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -3408,14 +3408,14 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "RSS",
               "refId": "A"
             },
             {
-              "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} \n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} \n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3423,7 +3423,7 @@
               "refId": "B"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left() \n    sum by (pod) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3431,35 +3431,35 @@
               "refId": "C"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Working set bytes without kmem",
               "refId": "D"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\",namespace=\"$namespace\"}[$__rate_interval]))\n  ) ",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by(pod) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))\n  ) ",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "E"
             },
             {
-              "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} \n    * on (pod) group_left() \n    sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
+              "expr": "sum\n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"} \n    * on (pod) group_left() \n    sum by(pod) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "F"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"memory\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (controller_type, controller_name) group_left()\n    sum by(controller_type, controller_name) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"memory\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
               "refId": "G"
             },
             {
-              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))\n  )",
+              "expr": "sum \n  (\n    kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller=\"$controller\"}\n    * on (pod) group_left()\n    sum by (pod) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))\n  )",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Kmem",
@@ -3910,7 +3910,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])))",
+          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_fs_reads_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval])))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",
@@ -4049,7 +4049,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])))",
+          "expr": "sum by (controller) (kube_controller_pod{node=~\"$node\", namespace=\"$namespace\", controller_type=~\"$controller_type\", controller=~\"$controller\"} * on (pod) group_left() sum by (pod) (rate(container_fs_writes_total{node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval])))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ controller }}",

dashboards/main/namespaces.json

@@ -869,7 +869,7 @@
           "refId": "A"
         },
         {
-          "expr": "100 * count by (namespace) (\n  sum by (namespace, verticalpodautoscaler) (  \n    count by (namespace, controller_name, verticalpodautoscaler) (avg_over_time(vpa_target_recommendation{namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\n    / on (controller_name, namespace) group_left\n    count by (namespace, controller_name) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range]))\n  )\n) \n/ count by (namespace) (sum by (namespace, controller) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "100 * count by (namespace) (\n  sum by (namespace, verticalpodautoscaler) (  \n    count by (namespace, controller_name, verticalpodautoscaler) (avg_over_time(vpa_target_recommendation{namespace=~\"$namespace\", container!=\"\"}[$__range]))\n    / on (controller_name, namespace) group_left\n    count by (namespace, controller_name) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range]))\n  )\n) \n/ count by (namespace) (sum by (namespace, controller) (avg_over_time(kube_controller_pod{namespace=~\"$namespace\"}[$__range])))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -878,7 +878,7 @@
           "refId": "B"
         },
         {
-          "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -895,7 +895,7 @@
           "refId": "D"
         },
         {
-          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n    ) > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n    ) > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -903,7 +903,7 @@
           "refId": "E"
         },
         {
-          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n    )\n    > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n    )\n    > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -919,7 +919,7 @@
           "refId": "G"
         },
         {
-          "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -935,7 +935,7 @@
           "refId": "I"
         },
         {
-          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n    ) > 0\n  )\nor\ncount(avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n    ) > 0\n  )\nor\ncount(avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -943,7 +943,7 @@
           "refId": "J"
         },
         {
-          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__range]))\n    )\n    > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__range]))\n      ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__range]))\n    )\n    > 0\n  )\nor count (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -968,7 +968,7 @@
           "refId": "M"
         },
         {
-          "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -977,7 +977,7 @@
           "refId": "N"
         },
         {
-          "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
+          "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__range]))\nor\ncount (avg_over_time(kube_controller_pod{node=~\"$node\", namespace=~\"$namespace\"}[$__range])) by (namespace) * 0",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -1449,7 +1449,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -1616,7 +1616,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "System",
@@ -1627,7 +1627,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "User",
@@ -1764,7 +1764,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -1901,7 +1901,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n    )\n    > 0\n  )",
+          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      ) or sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n    )\n    > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -2210,7 +2210,7 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -2218,21 +2218,21 @@
               "refId": "A"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})",
+              "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "B"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})",
+              "expr": "sum by (namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"cpu\",unit=\"core\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval])* on (uid) group_left(phase) kube_pod_status_phase{phase=\"Running\"})",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "C"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(vpa_target_recommendation{container!=\"POD\", namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(vpa_target_recommendation{container!=\"\", namespace=\"$namespace\", resource=\"cpu\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
@@ -2407,7 +2407,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by (namespace) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (rate(container_cpu_system_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -2419,7 +2419,7 @@
                 "type": "prometheus",
                 "uid": "$ds_prometheus"
               },
-              "expr": "sum by (namespace) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (rate(container_cpu_user_seconds_total{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "User",
@@ -2572,7 +2572,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -2754,14 +2754,14 @@
       "pluginVersion": "8.5.13",
       "targets": [
         {
-          "expr": "sum (avg_over_time(container_memory_rss{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory_rss{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "RSS",
           "refId": "A"
         },
         {
-          "expr": "sum (avg_over_time(container_memory_cache{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory_cache{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -2769,7 +2769,7 @@
           "refId": "B"
         },
         {
-          "expr": "sum (avg_over_time(container_memory_swap{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory_swap{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -2777,14 +2777,14 @@
           "refId": "C"
         },
         {
-          "expr": "sum (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Working set bytes without kmem",
           "refId": "D"
         },
         {
-          "expr": "sum (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Kmem",
@@ -2910,7 +2910,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (namespace)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -3046,7 +3046,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"POD\", namespace=~\"$namespace\"}[$__rate_interval]))\n    )\n    > 0\n  )",
+          "expr": "sum by (namespace)\n  (\n    (\n      (\n        sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n        -\n        sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", namespace=~\"$namespace\"}[$__rate_interval]))\n      ) or sum by(namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", container!=\"\", namespace=~\"$namespace\"}[$__rate_interval]))\n    )\n    > 0\n  )",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -3370,14 +3370,14 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory_rss{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "RSS",
               "refId": "A"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory_cache{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3385,7 +3385,7 @@
               "refId": "B"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory_swap{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3393,35 +3393,35 @@
               "refId": "C"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory_working_set_bytes:without_kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Working set bytes without kmem",
               "refId": "D"
             },
             {
-              "expr": "sum by(namespace) (avg_over_time(vpa_target_recommendation{container!=\"POD\",namespace=\"$namespace\", resource=\"memory\"}[$__rate_interval]))",
+              "expr": "sum by(namespace) (avg_over_time(vpa_target_recommendation{container!=\"\",namespace=\"$namespace\", resource=\"memory\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "VPA Target",
               "refId": "E"
             },
             {
-              "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
               "refId": "F"
             },
             {
-              "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"POD\", namespace=\"$namespace\"}[$__rate_interval]))",
+              "expr": "sum by(namespace) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",node=~\"$node\", container!=\"\", namespace=\"$namespace\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "G"
             },
             {
-              "expr": "sum by (namespace) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+              "expr": "sum by (namespace) (avg_over_time(container_memory:kmem{node=~\"$node\", namespace=\"$namespace\", container!=\"\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Kmem",
@@ -3873,7 +3873,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by (namespace) (rate(container_fs_reads_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",
@@ -4008,7 +4008,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by (namespace) (rate(container_fs_writes_total{node=~\"$node\", namespace=~\"$namespace\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ namespace }}",

dashboards/main/pod.json

@@ -686,7 +686,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
+          "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -759,7 +759,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
+          "expr": "sum by (container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -847,7 +847,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by(container) (rate(container_fs_reads_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__range]))",
+          "expr": "sum by(container) (rate(container_fs_reads_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__range]))",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -860,7 +860,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by(container) (rate(container_fs_writes_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__range]))",
+          "expr": "sum by(container) (rate(container_fs_writes_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__range]))",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -899,7 +899,7 @@
             "type": "prometheus",
             "uid": "${ds_prometheus}"
           },
-          "expr": "sum by (container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
+          "expr": "sum by (container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=~\"$container\"}[$__range]))\nor\nsum by (container) (avg_over_time(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__range]) * 0)",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -1503,7 +1503,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -1669,7 +1669,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(pod) (rate(container_cpu_system_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (rate(container_cpu_system_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -1681,7 +1681,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(pod) (rate(container_cpu_user_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (rate(container_cpu_user_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "User",
@@ -1820,7 +1820,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (namespace, pod, container)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{container!=\"POD\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (namespace, pod, container)\n  (\n    (\n      sum by(namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{container!=\"\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -2269,7 +2269,7 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Usage",
@@ -2476,7 +2476,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_cpu_system_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_cpu_system_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -2488,7 +2488,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_cpu_user_seconds_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_cpu_user_seconds_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\", container=\"$container\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "User",
@@ -2639,7 +2639,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -2816,7 +2816,7 @@
       "pluginVersion": "8.5.13",
       "targets": [
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -2824,28 +2824,28 @@
           "refId": "A"
         },
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Cache",
           "refId": "B"
         },
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Swap",
           "refId": "C"
         },
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Working set bytes without kmem",
           "refId": "D"
         },
         {
-          "expr": "sum by(pod) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}[$__rate_interval]))",
+          "expr": "sum by(pod) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Kmem",
@@ -2974,7 +2974,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (container)\n  (\n    (\n      sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (container)\n  (\n    (\n      sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n      -\n      sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -3110,7 +3110,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by (container)\n  (\n    (\n      (\n        sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n        -\n        sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))\n      ) or sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))\n    ) > 0\n  )",
+          "expr": "sum by (container)\n  (\n    (\n      (\n        sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"}[$__rate_interval]))\n        -\n        sum by (namespace, pod, container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"\"}[$__rate_interval]))\n      ) or sum by (namespace, pod, container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"\"}[$__rate_interval]))\n    ) > 0\n  )",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -3431,7 +3431,7 @@
           "repeatDirection": "h",
           "targets": [
             {
-              "expr": "sum by(container) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "instant": false,
               "intervalFactor": 1,
@@ -3439,7 +3439,7 @@
               "refId": "A"
             },
             {
-              "expr": "sum by(container) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3447,28 +3447,28 @@
               "refId": "B"
             },
             {
-              "expr": "sum by(container) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Swap",
               "refId": "C"
             },
             {
-              "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory_working_set_bytes:without_kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Working set bytes without kmem",
               "refId": "D"
             },
             {
-              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_limits{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Limits",
               "refId": "E"
             },
             {
-              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Requests",
@@ -3482,7 +3482,7 @@
               "refId": "G"
             },
             {
-              "expr": "sum by(container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container=\"$container\"}[$__rate_interval]))",
+              "expr": "sum by(container) (avg_over_time(container_memory:kmem{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", container=\"$container\"}[$__rate_interval]))",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "Kmem",
@@ -3930,7 +3930,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ container }}",
@@ -4068,7 +4068,7 @@
             "type": "prometheus",
             "uid": "$ds_prometheus"
           },
-          "expr": "sum by(container) (rate(container_fs_writes_total{container!=\"POD\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "expr": "sum by(container) (rate(container_fs_writes_total{container!=\"\", pod=\"$pod\", namespace=\"$namespace\"}[$__rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ container }}",

docs/changelogs/v0.31.0.md

@@ -0,0 +1,243 @@
+Cozystack v0.31.0 is a significant release that brings new features, key fixes, and updates to underlying components.
+This version enhances GPU support, improves many components of Cozystack, and introduces a more robust release process to improve stability.
+Below, we'll go over the highlights in each area for current users, developers, and our community.
+
+## Major Features and Improvements
+
+### GPU support for tenant Kubernetes clusters
+
+Cozystack now integrates NVIDIA GPU Operator support for tenant Kubernetes clusters.
+This enables platform users to run GPU-powered AI/ML applications in their own clusters.
+To enable GPU Operator, set `addons.gpuOperator.enabled: true` in the cluster configuration.
+(@kvaps in https://github.com/cozystack/cozystack/pull/834)
+
+Check out Andrei Kvapil's CNCF webinar [showcasing the GPU support by running Stable Diffusion in Cozystack](https://www.youtube.com/watch?v=S__h_QaoYEk).
+
+<!--
+* [kubernetes] Introduce GPU support for tenant Kubernetes clusters. (@kvaps in https://github.com/cozystack/cozystack/pull/834)
+-->
+
+### Cilium Improvements
+
+Cozystack’s Cilium integration received two significant enhancements.
+First, Gateway API support in Cilium is now enabled, allowing advanced L4/L7 routing features via Kubernetes Gateway API.
+We thank Zdenek Janda @zdenekjanda for contributing this feature in https://github.com/cozystack/cozystack/pull/924.
+
+Second, Cozystack now permits custom user-provided parameters in the tenant cluster’s Cilium configuration.
+(@lllamnyp in https://github.com/cozystack/cozystack/pull/917)
+
+<!--
+* [cilium] Enable Cilium Gateway API. (@zdenekjanda in https://github.com/cozystack/cozystack/pull/924)
+* [cilium] Enable user-added parameters in a tenant cluster Cilium. (@lllamnyp in https://github.com/cozystack/cozystack/pull/917)
+-->
+
+### Cross-Architecture Builds (ARM Support Beta)
+
+Cozystack's build system was refactored to support multi-architecture binaries and container images.
+This paves the road to running Cozystack on ARM64 servers.
+Changes include Makefile improvements (https://github.com/cozystack/cozystack/pull/907)
+and multi-arch Docker image builds (https://github.com/cozystack/cozystack/pull/932 and https://github.com/cozystack/cozystack/pull/970).
+
+We thank Nikita Bykov @nbykov0 for his ongoing work on ARM support!
+
+<!--
+* Introduce support for cross-architecture builds and Cozystack on ARM:
+    * [build] Refactor Makefiles introducing build variables. (@nbykov0 in https://github.com/cozystack/cozystack/pull/907)
+    * [build] Add support for multi-architecture and cross-platform image builds. (@nbykov0 in https://github.com/cozystack/cozystack/pull/932 and https://github.com/cozystack/cozystack/pull/970)
+-->
+
+### VerticalPodAutoscaler (VPA) Expansion
+
+The VerticalPodAutoscaler is now enabled for more Cozystack components to automate resource tuning.
+Specifically, VPA was added for tenant Kubernetes control planes (@klinch0 in https://github.com/cozystack/cozystack/pull/806),
+the Cozystack Dashboard (https://github.com/cozystack/cozystack/pull/828),
+and the Cozystack etcd-operator (https://github.com/cozystack/cozystack/pull/850).
+All Cozystack components that have VPA enabled can automatically adjust their CPU and memory requests based on usage, improving platform and application stability.
+
+<!--
+* Add VerticalPodAutoscaler to a few more components:
+    * [kubernetes] Kubernetes clusters in user tenants. (@klinch0 in https://github.com/cozystack/cozystack/pull/806)
+    * [platform] Cozystack dashboard. (@klinch0 in https://github.com/cozystack/cozystack/pull/828)
+    * [platform] Cozystack etcd-operator (@klinch0 in https://github.com/cozystack/cozystack/pull/850)
+-->
+
+### Tenant HelmRelease Reconcile Controller
+
+A new controller was introduced to monitor and synchronize HelmRelease resources across tenants.
+This controller propagates configuration changes to tenant workloads and ensures that any HelmRelease defined in a tenant
+stays in sync with platform updates.
+It improves the reliability of deploying managed applications in Cozystack.
+(@klinch0 in https://github.com/cozystack/cozystack/pull/870)
+
+<!--
+* [platform] Introduce a new controller to synchronize tenant HelmReleases and propagate configuration changes. (@klinch0 in https://github.com/cozystack/cozystack/pull/870)
+-->
+
+### Virtual Machine Improvements
+
+**Configurable KubeVirt CPU Overcommit**: The CPU allocation ratio in KubeVirt (how virtual CPUs are overcommitted relative to physical) is now configurable
+via the `cpu-allocation-ratio` value in the Cozystack configmap.
+This means Cozystack administrators can now tune CPU overcommitment for VMs to balance performance vs. density.
+(@lllamnyp in https://github.com/cozystack/cozystack/pull/905)
+
+**KubeVirt VM Export**: Cozystack now allows exporting KubeVirt virtual machines.
+This feature, enabled via KubeVirt's `VirtualMachineExport` capability, lets users snapshot or back up VM images.
+(@kvaps in https://github.com/cozystack/cozystack/pull/808)
+
+**Support for various storage classes in Virtual Machines**: The `virtual-machine` application (since version 0.9.2) lets you pick any StorageClass for a VM's 
+system disk instead of relying on a hard-coded PVC.
+Refer to values `systemDisk.storage` and `systemDisk.storageClass` in the [application's configs](https://cozystack.io/docs/reference/applications/virtual-machine/#common-parameters).
+(@kvaps in https://github.com/cozystack/cozystack/pull/974)
+
+<!--
+* [kubevirt] Enable exporting VMs. (@kvaps in https://github.com/cozystack/cozystack/pull/808)
+* [kubevirt] Make KubeVirt's CPU allocation ratio configurable. (@lllamnyp in https://github.com/cozystack/cozystack/pull/905)
+* [virtual-machine] Add support for various storages. (@kvaps in https://github.com/cozystack/cozystack/pull/974)
+-->
+
+### Other Features and Improvements
+
+* [platform] Introduce options `expose-services`, `expose-ingress`, and `expose-external-ips` to the ingress service. (@kvaps in https://github.com/cozystack/cozystack/pull/929)
+* [cozystack-controller] Record the IP address pool and storage class in Workload objects. (@lllamnyp in https://github.com/cozystack/cozystack/pull/831)
+* [apps] Remove user-facing config of limits and requests. (@lllamnyp in https://github.com/cozystack/cozystack/pull/935)
+
+## New Release Lifecycle
+
+Cozystack release lifecycle is changing to provide a more stable and predictable lifecycle to customers running Cozystack in mission-critical environments.
+
+* **Gradual Release with Alpha, Beta, and Release Candidates**: Cozystack will now publish pre-release versions (alpha, beta, release candidates) before a stable release.
+  Starting with v0.31.0, the team made three release candidates before releasing version v0.31.0.
+  This allows more testing and feedback before marking a release as stable.
+
+* **Prolonged Release Support with Patch Versions**: After the initial `vX.Y.0` release, a long-lived branch `release-X.Y` will be created to backport fixes.
+  For example, with 0.31.0’s release, a `release-0.31` branch will track patch fixes (`0.31.x`).
+  This strategy lets Cozystack users receive timely patch releases and updates with minimal risks.
+
+To implement these new changes, we have rebuilt our CI/CD workflows and introduced automation, enabling automatic backports.
+You can read more about how it's implemented in the Development section below.
+
+For more information, read the [Cozystack Release Workflow](https://github.com/cozystack/cozystack/blob/main/docs/release.md) documentation.
+
+## Fixes
+
+* [virtual-machine] Add GPU names to the virtual machine specifications. (@kvaps in https://github.com/cozystack/cozystack/pull/862)
+* [virtual-machine] Count Workload resources for pods by requests, not limits. Other improvements to VM resource tracking. (@lllamnyp in https://github.com/cozystack/cozystack/pull/904)
+* [virtual-machine] Set PortList method by default. (@kvaps in https://github.com/cozystack/cozystack/pull/996)
+* [virtual-machine] Specify ports even for wholeIP mode. (@kvaps in https://github.com/cozystack/cozystack/pull/1000)
+* [platform] Fix installing HelmReleases on initial setup. (@kvaps in https://github.com/cozystack/cozystack/pull/833)
+* [platform] Migration scripts update Kubernetes ConfigMap with the current stack version for improved version tracking. (@klinch0 in https://github.com/cozystack/cozystack/pull/840)
+* [platform] Reduce requested CPU and RAM for the `kamaji` provider. (@klinch0 in https://github.com/cozystack/cozystack/pull/825)
+* [platform] Improve the reconciliation loop for the Cozystack system HelmReleases logic. (@klinch0 in https://github.com/cozystack/cozystack/pull/809 and https://github.com/cozystack/cozystack/pull/810, @kvaps in https://github.com/cozystack/cozystack/pull/811)
+* [platform] Remove extra dependencies for the Piraeus operator. (@klinch0 in https://github.com/cozystack/cozystack/pull/856)
+* [platform] Refactor dashboard values. (@kvaps in https://github.com/cozystack/cozystack/pull/928, patched by @llamnyp in https://github.com/cozystack/cozystack/pull/952)
+* [platform] Make FluxCD artifact disabled by default. (@klinch0 in https://github.com/cozystack/cozystack/pull/964)
+* [kubernetes] Update garbage collection of HelmReleases in tenant Kubernetes clusters. (@kvaps in https://github.com/cozystack/cozystack/pull/835)
+* [kubernetes] Fix merging `valuesOverride` for tenant clusters. (@kvaps in https://github.com/cozystack/cozystack/pull/879)
+* [kubernetes] Fix `ubuntu-container-disk` tag. (@kvaps in https://github.com/cozystack/cozystack/pull/887)
+* [kubernetes] Refactor Helm manifests for tenant Kubernetes clusters. (@kvaps in https://github.com/cozystack/cozystack/pull/866)
+* [kubernetes] Fix Ingress-NGINX depends on Cert-Manager. (@kvaps in https://github.com/cozystack/cozystack/pull/976)
+* [kubernetes, apps] Enable `topologySpreadConstraints` for tenant Kubernetes clusters and fix it for managed PostgreSQL. (@klinch0 in https://github.com/cozystack/cozystack/pull/995)
+* [tenant] Fix an issue with accessing external IPs of a cluster from the cluster itself. (@kvaps in https://github.com/cozystack/cozystack/pull/854)
+* [cluster-api] Remove the no longer necessary workaround for Kamaji. (@kvaps in https://github.com/cozystack/cozystack/pull/867, patched in https://github.com/cozystack/cozystack/pull/956) 
+* [monitoring] Remove legacy label "POD" from the exclude filter in metrics. (@xy2 in https://github.com/cozystack/cozystack/pull/826)
+* [monitoring] Refactor management etcd monitoring config. Introduce a migration script for updating monitoring resources (`kube-rbac-proxy` daemonset). (@lllamnyp in https://github.com/cozystack/cozystack/pull/799 and https://github.com/cozystack/cozystack/pull/830)
+* [monitoring] Fix VerticalPodAutoscaler resource allocation for VMagent. (@klinch0 in https://github.com/cozystack/cozystack/pull/820)
+* [postgres] Remove duplicated `template` entry from backup manifest. (@etoshutka in https://github.com/cozystack/cozystack/pull/872)
+* [kube-ovn] Fix versions mapping in Makefile. (@kvaps in https://github.com/cozystack/cozystack/pull/883)
+* [dx] Automatically detect version for migrations in the installer.sh. (@kvaps in https://github.com/cozystack/cozystack/pull/837)
+* [dx] remove version_map and building for library charts. (@kvaps in https://github.com/cozystack/cozystack/pull/998)
+* [docs] Review the tenant Kubernetes cluster docs. (@NickVolynkin in https://github.com/cozystack/cozystack/pull/969)
+* [docs] Explain that tenants cannot have dashes in their names. (@NickVolynkin in https://github.com/cozystack/cozystack/pull/980)
+
+## Dependencies
+
+* MetalLB images are now built in-tree based on version 0.14.9 with additional critical patches. (@lllamnyp in https://github.com/cozystack/cozystack/pull/945)
+* Update Kubernetes to v1.32.4. (@kvaps in https://github.com/cozystack/cozystack/pull/949)
+* Update Talos Linux to v1.10.1. (@kvaps in https://github.com/cozystack/cozystack/pull/931)
+* Update Cilium to v1.17.3. (@kvaps in https://github.com/cozystack/cozystack/pull/848)
+* Update LINSTOR to v1.31.0. (@kvaps in https://github.com/cozystack/cozystack/pull/846)
+* Update Kube-OVN to v1.13.11. (@kvaps in https://github.com/cozystack/cozystack/pull/847, @lllamnyp in https://github.com/cozystack/cozystack/pull/922)
+* Update tenant Kubernetes to v1.32. (@kvaps in https://github.com/cozystack/cozystack/pull/871)
+* Update flux-operator to 0.20.0. (@kingdonb in https://github.com/cozystack/cozystack/pull/880 and https://github.com/cozystack/cozystack/pull/934)
+* Update multiple Cluster API components. (@kvaps in https://github.com/cozystack/cozystack/pull/867 and https://github.com/cozystack/cozystack/pull/947)
+* Update KamajiControlPlane to edge-25.4.1. (@kvaps in https://github.com/cozystack/cozystack/pull/953, fixed by @nbykov0 in https://github.com/cozystack/cozystack/pull/983)
+* Update cert-manager to v1.17.2. (@kvaps in https://github.com/cozystack/cozystack/pull/975)
+
+## Documentation
+
+* [Installing Talos in Air-Gapped Environment](https://cozystack.io/docs/operations/talos/configuration/air-gapped/):
+  new guide for configuring and bootstrapping Talos Linux clusters in air-gapped environments.
+  (@klinch0 in https://github.com/cozystack/website/pull/203)
+
+* [Cozystack Bundles](https://cozystack.io/docs/guides/bundles/): new page in the learning section explaining how Cozystack bundles work and how to choose a bundle.
+  (@NickVolynkin in https://github.com/cozystack/website/pull/188, https://github.com/cozystack/website/pull/189, and others;
+  updated by @kvaps in https://github.com/cozystack/website/pull/192 and https://github.com/cozystack/website/pull/193)
+
+* [Managed Application Reference](https://cozystack.io/docs/reference/applications/): A set of new pages in the docs, mirroring application docs from the Cozystack dashboard.
+  (@NickVolynkin in https://github.com/cozystack/website/pull/198, https://github.com/cozystack/website/pull/202, and https://github.com/cozystack/website/pull/204)
+
+* **LINSTOR Networking**: Guides on [configuring dedicated network for LINSTOR](https://cozystack.io/docs/operations/storage/dedicated-network/)
+and [configuring network for distributed storage in multi-datacenter setup](https://cozystack.io/docs/operations/stretched/linstor-dedicated-network/).
+(@xy2, edited by @NickVolynkin in https://github.com/cozystack/website/pull/171, https://github.com/cozystack/website/pull/182, and https://github.com/cozystack/website/pull/184)
+
+### Fixes
+
+* Correct error in the doc for the command to edit the configmap. (@lb0o in https://github.com/cozystack/website/pull/207)
+* Fix group name in OIDC docs (@kingdonb in https://github.com/cozystack/website/pull/179)
+* A bit more explanation of Docker buildx builders. (@nbykov0 in https://github.com/cozystack/website/pull/187)
+
+## Development, Testing, and CI/CD
+
+### Testing
+
+Improvements:
+
+* Introduce `cozytest` — a new [BATS-based](https://github.com/bats-core/bats-core) testing framework. (@kvaps in https://github.com/cozystack/cozystack/pull/982)
+
+Fixes:
+
+* Fix `device_ownership_from_security_context` CRI. (@dtrdnk in https://github.com/cozystack/cozystack/pull/896)
+* Increase timeout durations for `capi` and `keycloak` to improve reliability during e2e-tests. (@kvaps in https://github.com/cozystack/cozystack/pull/858)
+* Return `genisoimage` to the e2e-test Dockerfile (@gwynbleidd2106 in https://github.com/cozystack/cozystack/pull/962)
+
+### CI/CD Changes
+                          
+Improvements:
+
+* Use release branches `release-X.Y` for gathering and releasing fixes after initial `vX.Y.0` release. (@kvaps in https://github.com/cozystack/cozystack/pull/816)
+* Automatically create release branches after initial `vX.Y.0` release is published. (@kvaps in https://github.com/cozystack/cozystack/pull/886)
+* Introduce Release Candidate versions. Automate patch backporting by applying patches from pull requests labeled `[backport]` to the current release branch. (@kvaps in https://github.com/cozystack/cozystack/pull/841 and https://github.com/cozystack/cozystack/pull/901, @nickvolynkin in https://github.com/cozystack/cozystack/pull/890)
+* Support alpha and beta pre-releases. (@kvaps in https://github.com/cozystack/cozystack/pull/978)
+* Commit changes in release pipelines under `github-actions <github-actions@github.com>`. (@kvaps in https://github.com/cozystack/cozystack/pull/823)
+* Describe the Cozystack release workflow. (@NickVolynkin in https://github.com/cozystack/cozystack/pull/817 and https://github.com/cozystack/cozystack/pull/897)
+
+Fixes:
+
+* Improve the check for `versions_map` running on pull requests. (@kvaps and @klinch0 in https://github.com/cozystack/cozystack/pull/836, https://github.com/cozystack/cozystack/pull/842, and https://github.com/cozystack/cozystack/pull/845)
+* If the release step was skipped on a tag, skip tests as well. (@kvaps in https://github.com/cozystack/cozystack/pull/822)
+* Allow CI to cancel the previous job if a new one is scheduled. (@kvaps in https://github.com/cozystack/cozystack/pull/873)
+* Use the correct version name when uploading build assets to the release page. (@kvaps in https://github.com/cozystack/cozystack/pull/876)
+* Stop using `ok-to-test` label to trigger CI in pull requests. (@kvaps in https://github.com/cozystack/cozystack/pull/875)
+* Do not run tests in the release building pipeline. (@kvaps in https://github.com/cozystack/cozystack/pull/882)
+* Fix release branch creation. (@kvaps in https://github.com/cozystack/cozystack/pull/884)
+* Reduce noise in the test logs by suppressing the `wget` progress bar. (@lllamnyp in https://github.com/cozystack/cozystack/pull/865)
+* Revert "automatically trigger tests in releasing PR". (@kvaps in https://github.com/cozystack/cozystack/pull/900)
+* Force-update release branch on tagged main commits. (@kvaps in https://github.com/cozystack/cozystack/pull/977)
+* Show detailed errors in the `pull-request-release` workflow. (@lllamnyp in https://github.com/cozystack/cozystack/pull/992)
+
+## Community and Maintenance
+
+### Repository Maintenance
+
+Added @klinch0 to CODEOWNERS. (@kvaps in https://github.com/cozystack/cozystack/pull/838)
+
+### New Contributors
+
+* @etoshutka made their first contribution in https://github.com/cozystack/cozystack/pull/872
+* @dtrdnk made their first contribution in https://github.com/cozystack/cozystack/pull/896
+* @zdenekjanda made their first contribution in https://github.com/cozystack/cozystack/pull/924
+* @gwynbleidd2106 made their first contribution in https://github.com/cozystack/cozystack/pull/962
+
+## Full Changelog
+
+See https://github.com/cozystack/cozystack/compare/v0.30.0...v0.31.0

docs/changelogs/v0.31.1.md

@@ -0,0 +1,8 @@
+## Fixes
+
+* [build] Update Talos Linux v1.10.3 and fix assets. (@kvaps in https://github.com/cozystack/cozystack/pull/1006)
+* [ci] Fix uploading released artifacts to GitHub. (@kvaps in https://github.com/cozystack/cozystack/pull/1009)
+* [ci] Separate build and testing jobs. (@kvaps in https://github.com/cozystack/cozystack/pull/1005)
+* [docs] Write a full release post for v0.31.1. (@NickVolynkin in https://github.com/cozystack/cozystack/pull/999)
+
+**Full Changelog**: https://github.com/cozystack/cozystack/compare/v0.31.0...v0.31.1

docs/changelogs/v0.31.2.md

@@ -0,0 +1,13 @@
+## Security
+
+* Resolve a security problem that allowed a tenant administrator to gain enhanced privileges outside the tenant. (@kvaps in https://github.com/cozystack/cozystack/pull/1062, backported in https://github.com/cozystack/cozystack/pull/1066)
+
+## Fixes
+
+* [platform] Fix dependencies in `distro-full` bundle. (@klinch0 in  https://github.com/cozystack/cozystack/pull/1056, backported in https://github.com/cozystack/cozystack/pull/1064)
+* [platform] Fix RBAC for annotating namespaces. (@kvaps in https://github.com/cozystack/cozystack/pull/1031, backported in https://github.com/cozystack/cozystack/pull/1037)
+* [platform] Reduce system resource consumption by using smaller resource presets for VerticalPodAutoscaler, SeaweedFS, and KubeOVN. (@klinch0 in https://github.com/cozystack/cozystack/pull/1054, backported in https://github.com/cozystack/cozystack/pull/1058)
+* [dashboard] Fix a number of issues in the Cozystack Dashboard (@kvaps in https://github.com/cozystack/cozystack/pull/1042, backported in https://github.com/cozystack/cozystack/pull/1066)
+* [apps] Specify minimal working resource presets. (@kvaps in https://github.com/cozystack/cozystack/pull/1040, backported in https://github.com/cozystack/cozystack/pull/1041)
+* [apps] Update built-in documentation and configuration reference for managed Clickhouse application. (@NickVolynkin in https://github.com/cozystack/cozystack/pull/1059, backported in https://github.com/cozystack/cozystack/pull/1065)
+                                                                          ы

docs/release.md

@@ -0,0 +1,166 @@
+# Release Workflow
+
+This document describes Cozystack’s release process.
+
+## Introduction
+
+Cozystack uses a staged release process to ensure stability and flexibility during development.
+
+There are three types of releases:
+
+- **Release Candidates (RC)** – Preview versions (e.g., `v0.42.0-rc.1`) used for final testing and validation.
+- **Regular Releases** – Final versions (e.g., `v0.42.0`) that are feature-complete and thoroughly tested.
+- **Patch Releases** – Bugfix-only updates (e.g., `v0.42.1`) made after a stable release, based on a dedicated release branch.
+
+Each type plays a distinct role in delivering reliable and tested updates while allowing ongoing development to continue smoothly.
+
+## Release Candidates
+
+Release candidates are Cozystack versions that introduce new features and are published before a stable release.
+Their purpose is to help validate stability before finalizing a new feature release.
+They allow for final rounds of testing and bug fixes without freezing development.
+
+Release candidates are given numbers `vX.Y.0-rc.N`, for example, `v0.42.0-rc.1`.
+They are created directly in the `main` branch.
+An RC is typically tagged when all major features for the upcoming release have been merged into main and the release enters its testing phase.
+However, new features and changes can still be added before the regular release `vX.Y.0`.
+
+Each RC contributes to a cumulative set of release notes that will be finalized when `vX.Y.0` is released.
+After testing, if no critical issues remain, the regular release (`vX.Y.0`) is tagged from the last RC or a later commit in main.
+This begins the regular release process, creates a dedicated `release-X.Y` branch, and opens the way for patch releases.
+
+## Regular Releases
+
+When making a regular release, we tag the latest RC or a subsequent minimal-change commit as `vX.Y.0`.
+In this explanation, we'll use version `v0.42.0` as an example:
+
+```mermaid
+gitGraph
+    commit id: "feature"
+    commit id: "feature 2"
+    commit id: "feature 3" tag: "v0.42.0"
+```
+
+A regular release sequence starts in the following way:
+
+1. Maintainer tags a commit in `main` with `v0.42.0` and pushes it to GitHub.
+2. CI workflow triggers on tag push:
+   1. Creates a draft page for release `v0.42.0`, if it wasn't created before.
+   2. Takes code from tag `v0.42.0`, builds images, and pushes them to ghcr.io.
+   3. Makes a new commit `Prepare release v0.42.0` with updated digests, pushes it to the new branch `release-0.42.0`, and opens a PR to `main`.
+   4. Builds Cozystack release assets from the new commit `Prepare release v0.42.0` and uploads them to the release draft page.
+3. Maintainer reviews PR, tests build artifacts, and edits changelogs on the release draft page.
+
+   ```mermaid
+   gitGraph
+       commit id: "feature"
+       commit id: "feature 2"
+       commit id: "feature 3" tag: "v0.42.0"
+       branch release-0.42.0
+       checkout release-0.42.0
+       commit id: "Prepare release v0.42.0"
+       checkout main
+       merge release-0.42.0 id: "Pull Request"
+   ```
+
+   When testing and editing are completed, the sequence goes on.
+
+4. Maintainer merges the PR. GitHub removes the merged branch `release-0.42.0`.
+5. CI workflow triggers on merge:
+   1. Moves the tag `v0.42.0` to the newly created merge commit by force-pushing a tag to GitHub.
+   2. Publishes the release page (`draft` → `latest`).
+6. The maintainer can now announce the release to the community.
+
+```mermaid
+gitGraph
+    commit id: "feature"
+    commit id: "feature 2"
+    commit id: "feature 3"
+    branch release-0.42.0
+    checkout release-0.42.0
+    commit id: "Prepare release v0.42.0"
+    checkout main
+    merge release-0.42.0 id: "Release v0.42.0" tag: "v0.42.0"
+```
+
+## Patch Releases
+
+Making a patch release has a lot in common with a regular release, with a couple of differences:
+
+* A release branch is used instead of `main`
+* Patch commits are cherry-picked to the release branch.
+* A pull request is opened against the release branch.
+
+
+Let's assume that we've released `v0.42.0` and that development is ongoing.
+We have introduced a couple of new features and some fixes to features that we have released 
+in `v0.42.0`.
+
+Once problems were found and fixed, a patch release is due.
+
+```mermaid
+gitGraph
+   commit id: "Release v0.42.0" tag: "v0.42.0"
+    checkout main
+    commit id: "feature 4"
+    commit id: "patch 1"
+    commit id: "feature 5"
+    commit id: "patch 2"
+```
+
+
+1. The maintainer creates a release branch, `release-0.42,` and cherry-picks patch commits from `main` to `release-0.42`.
+   These must be only patches to features that were present in version `v0.42.0`.
+
+   Cherry-picking can be done as soon as each patch is merged into `main`,
+   or directly before the release.
+
+   ```mermaid
+   gitGraph
+       commit id: "Release v0.42.0" tag: "v0.42.0"
+       branch release-0.42
+       checkout main
+       commit id: "feature 4"
+       commit id: "patch 1"
+       commit id: "feature 5"
+       commit id: "patch 2"
+       checkout release-0.42
+       cherry-pick id: "patch 1"
+       cherry-pick id: "patch 2"
+   ```
+
+   When all relevant patch commits are cherry-picked, the branch is ready for release.
+
+2. The maintainer tags the `HEAD` commit of branch `release-0.42` as `v0.42.1` and then pushes it to GitHub.
+3. CI workflow triggers on tag push:
+    1. Creates a draft page for release `v0.42.1`, if it wasn't created before.
+    2. Takes code from tag `v0.42.1`, builds images, and pushes them to ghcr.io.
+    3. Makes a new commit `Prepare release v0.42.1` with updated digests, pushes it to the new branch `release-0.42.1`, and opens a PR to `release-0.42`.
+    4. Builds Cozystack release assets from the new commit `Prepare release v0.42.1` and uploads them to the release draft page.
+4. Maintainer reviews PR, tests build artifacts, and edits changelogs on the release draft page.
+   
+   ```mermaid
+   gitGraph
+       commit id: "Release v0.42.0" tag: "v0.42.0"
+       branch release-0.42
+       checkout main
+       commit id: "feature 4"
+       commit id: "patch 1"
+       commit id: "feature 5"
+       commit id: "patch 2"
+       checkout release-0.42
+       cherry-pick id: "patch 1"
+       cherry-pick id: "patch 2" tag: "v0.42.1"
+       branch release-0.42.1
+       commit id: "Prepare release v0.42.1"
+       checkout release-0.42
+       merge release-0.42.1 id: "Pull request"
+   ```
+
+   Finally, when release is confirmed, the release sequence goes on.
+
+5. Maintainer merges the PR. GitHub removes the merged branch `release-0.42.1`.
+6. CI workflow triggers on merge:
+   1. Moves the tag `v0.42.1` to the newly created merge commit by force-pushing a tag to GitHub.
+   2. Publishes the release page (`draft` → `latest`).
+7. The maintainer can now announce the release to the community.

hack/cozytest.sh

@@ -0,0 +1,117 @@
+#!/bin/sh
+###############################################################################
+# cozytest.sh - Bats-compatible test runner with live trace and enhanced      #
+# output, written in pure shell                                               #
+###############################################################################
+set -eu
+
+TEST_FILE=${1:?Usage: ./cozytest.sh <file.bats> [pattern]}
+PATTERN=${2:-*}
+LINE='----------------------------------------------------------------'
+
+cols() { stty size 2>/dev/null | awk '{print $2}' || echo 80; }
+MAXW=$(( $(cols) - 12 )); [ "$MAXW" -lt 40 ] && MAXW=70
+BEGIN=$(date +%s)
+timestamp() { s=$(( $(date +%s) - BEGIN )); printf '[%02d:%02d]' $((s/60)) $((s%60)); }
+
+###############################################################################
+# run_one <fn> <title>                                                        #
+###############################################################################
+run_one() {
+  fn=$1 title=$2
+  tmp=$(mktemp -d) || { echo "Failed to create temp directory" >&2; exit 1; }
+  log="$tmp/log"
+
+  echo "╭ » Run test: $title"
+  START=$(date +%s)
+  skip_next="+ $fn"      # первую строку трассировки с именем функции пропустим
+
+  {
+    (
+      PS4='+ '           # prefix for set -x
+      set -eu -x         # strict + trace
+      "$fn"
+    )
+    printf '__RC__%s\n' "$?"
+  } 2>&1 | tee "$log" | while IFS= read -r line; do
+        case "$line" in
+          '__RC__'*) : ;;
+          '+ '*)   cmd=${line#'+ '}
+                    [ "$cmd" = "${skip_next#+ }" ] && continue
+                    case "$cmd" in
+                      'set -e'|'set -x'|'set -u'|'return 0') continue ;;
+                    esac
+                    out=$cmd ;;
+          *)       out=$line ;;
+        esac
+        now=$(( $(date +%s) - START ))
+        [ ${#out} -gt "$MAXW" ] && out="$(printf '%.*s…' "$MAXW" "$out")"
+        printf '┊[%02d:%02d] %s\n' $((now/60)) $((now%60)) "$out"
+  done
+
+  rc=$(awk '/^__RC__/ {print substr($0,7)}' "$log" | tail -n1)
+  [ -z "$rc" ] && rc=1
+  now=$(( $(date +%s) - START ))
+
+  if [ "$rc" -eq 0 ]; then
+    printf '╰[%02d:%02d] ✅ Test OK: %s\n' $((now/60)) $((now%60)) "$title"
+  else
+    printf '╰[%02d:%02d] ❌ Test failed: %s (exit %s)\n' \
+           $((now/60)) $((now%60)) "$title" "$rc"
+    echo "----- captured output -----------------------------------------"
+    grep -v '^__RC__' "$log"
+    echo "$LINE"
+    exit "$rc"
+  fi
+
+  rm -rf "$tmp"
+}
+
+###############################################################################
+# convert .bats -> shell-functions                                            #
+###############################################################################
+TMP_SH=$(mktemp) || { echo "Failed to create temp file" >&2; exit 1; }
+trap 'rm -f "$TMP_SH"' EXIT
+awk '
+  /^@test[[:space:]]+"/ {
+    line  = substr($0, index($0, "\"") + 1)
+    title = substr(line, 1, index(line, "\"") - 1)
+    fname = "test_"
+    for (i = 1; i <= length(title); i++) {
+      c = substr(title, i, 1)
+      fname = fname (c ~ /[A-Za-z0-9]/ ? c : "_")
+    }
+    printf("### %s\n", title)
+    printf("%s() {\n", fname)
+    print "  set -e"           # ошибка → падение теста
+    next
+  }
+  /^}$/ {
+    print "  return 0"         # если автор не сделал exit 1 — тест ОК
+    print "}"
+    next
+  }
+  { print }
+' "$TEST_FILE" > "$TMP_SH"
+
+[ -f "$TMP_SH" ] || { echo "Failed to generate test functions" >&2; exit 1; }
+# shellcheck disable=SC1090
+. "$TMP_SH"
+
+###############################################################################
+# run selected tests                                                          #
+###############################################################################
+awk -v pat="$PATTERN" '
+  /^### / {
+    title = substr($0, 5)
+    name = "test_"
+    for (i = 1; i <= length(title); i++) {
+      c = substr(title, i, 1)
+      name = name (c ~ /[A-Za-z0-9]/ ? c : "_")
+    }
+    if (pat == "*" || index(title, pat) > 0)
+      printf("%s %s\n", name, title)
+  }
+' "$TMP_SH" | while IFS=' ' read -r fn title; do
+  run_one "$fn" "$title"
+done

hack/e2e-apps.bats

@@ -0,0 +1,94 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# Cozystack end‑to‑end provisioning test (Bats)
+# -----------------------------------------------------------------------------
+
+@test "Create tenant with isolated mode enabled" {
+  kubectl create -f - <<EOF
+apiVersion: apps.cozystack.io/v1alpha1
+kind: Tenant
+metadata:
+  name: test
+  namespace: tenant-root
+spec:
+  etcd: false
+  host: ""
+  ingress: false
+  isolated: true
+  monitoring: false
+  resourceQuotas: {}
+  seaweedfs: false
+EOF
+  kubectl wait hr/tenant-test -n tenant-root --timeout=1m --for=condition=ready
+  kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
+}
+
+@test "Create a tenant Kubernetes control plane" {
+  kubectl create -f - <<EOF
+apiVersion: apps.cozystack.io/v1alpha1
+kind: Kubernetes
+metadata:
+  name: test
+  namespace: tenant-test
+spec:
+  addons:
+    certManager:
+      enabled: false
+      valuesOverride: {}
+    cilium:
+      valuesOverride: {}
+    fluxcd:
+      enabled: false
+      valuesOverride: {}
+    gatewayAPI:
+      enabled: false
+    gpuOperator:
+      enabled: false
+      valuesOverride: {}
+    ingressNginx:
+      enabled: true
+      hosts: []
+      valuesOverride: {}
+    monitoringAgents:
+      enabled: false
+      valuesOverride: {}
+    verticalPodAutoscaler:
+      valuesOverride: {}
+  controlPlane:
+    apiServer:
+      resources: {}
+      resourcesPreset: small
+    controllerManager:
+      resources: {}
+      resourcesPreset: micro
+    konnectivity:
+      server:
+        resources: {}
+        resourcesPreset: micro
+    replicas: 2
+    scheduler:
+      resources: {}
+      resourcesPreset: micro
+  host: ""
+  nodeGroups:
+    md0:
+      ephemeralStorage: 20Gi
+      gpus: []
+      instanceType: u1.medium
+      maxReplicas: 10
+      minReplicas: 0
+      resources:
+        cpu: ""
+        memory: ""
+      roles:
+      - ingress-nginx
+  storageClass: replicated
+EOF
+  kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
+  timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test; do sleep 1; done'
+  kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test --timeout=4m
+  kubectl wait tcp -n tenant-test kubernetes-test --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
+  kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test kubernetes-test-cluster-autoscaler kubernetes-test-kccm kubernetes-test-kcsi-controller
+  kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
+  kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
+}

hack/e2e-cluster.bats

@@ -0,0 +1,391 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# Cozystack end‑to‑end provisioning test (Bats)
+# -----------------------------------------------------------------------------
+
+@test "Required installer assets exist" {
+  if [ ! -f _out/assets/cozystack-installer.yaml ]; then
+    echo "Missing: _out/assets/cozystack-installer.yaml" >&2
+    exit 1
+  fi
+
+  if [ ! -f _out/assets/nocloud-amd64.raw.xz ]; then
+    echo "Missing: _out/assets/nocloud-amd64.raw.xz" >&2
+    exit 1
+  fi
+}
+
+@test "IPv4 forwarding is enabled" {
+  if [ "$(cat /proc/sys/net/ipv4/ip_forward)" != 1 ]; then
+    echo "IPv4 forwarding is disabled!" >&2
+    echo >&2
+    echo "Enable it with:" >&2
+    echo "  echo 1 > /proc/sys/net/ipv4/ip_forward" >&2
+    exit 1
+  fi
+}
+
+@test "Clean previous VMs" {
+ kill $(cat srv1/qemu.pid srv2/qemu.pid srv3/qemu.pid 2>/dev/null) 2>/dev/null || true
+ rm -rf srv1 srv2 srv3
+}
+
+@test "Prepare networking and masquerading" {
+  ip link del cozy-br0 2>/dev/null || true
+  ip link add cozy-br0 type bridge
+  ip link set cozy-br0 up
+  ip address add 192.168.123.1/24 dev cozy-br0
+
+  # Masquerading rule – idempotent (delete first, then add)
+  iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE 2>/dev/null || true
+  iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE
+}
+
+@test "Prepare cloud‑init drive for VMs" {
+  mkdir -p srv1 srv2 srv3
+
+  # Generate cloud‑init ISOs
+  for i in 1 2 3; do
+    echo "hostname: srv${i}" > "srv${i}/meta-data"
+
+    cat > "srv${i}/user-data" <<'EOF'
+#cloud-config
+EOF
+
+    cat > "srv${i}/network-config" <<EOF
+version: 2
+ethernets:
+  eth0:
+    dhcp4: false
+    addresses:
+      - "192.168.123.1${i}/26"
+    gateway4: "192.168.123.1"
+    nameservers:
+      search: [cluster.local]
+      addresses: [8.8.8.8]
+EOF
+
+    ( cd "srv${i}" && genisoimage \
+        -output seed.img \
+        -volid cidata -rational-rock -joliet \
+        user-data meta-data network-config )
+  done
+}
+
+@test "Use Talos NoCloud image from assets" {
+  if [ ! -f _out/assets/nocloud-amd64.raw.xz ]; then
+    echo "Missing _out/assets/nocloud-amd64.raw.xz" 2>&1
+    exit 1
+  fi
+
+  rm -f nocloud-amd64.raw
+  cp _out/assets/nocloud-amd64.raw.xz .
+  xz --decompress nocloud-amd64.raw.xz
+}
+
+@test "Prepare VM disks" {
+  for i in 1 2 3; do
+    cp nocloud-amd64.raw srv${i}/system.img
+    qemu-img resize srv${i}/system.img 50G
+    qemu-img create srv${i}/data.img 100G
+  done
+}
+
+@test "Create tap devices" {
+  for i in 1 2 3; do
+    ip link del cozy-srv${i} 2>/dev/null || true
+    ip tuntap add dev cozy-srv${i} mode tap
+    ip link set cozy-srv${i} up
+    ip link set cozy-srv${i} master cozy-br0
+  done
+}
+
+@test "Boot QEMU VMs" {
+  for i in 1 2 3; do
+    qemu-system-x86_64 -machine type=pc,accel=kvm -cpu host -smp 8 -m 16384 \
+      -device virtio-net,netdev=net0,mac=52:54:00:12:34:5${i} \
+      -netdev tap,id=net0,ifname=cozy-srv${i},script=no,downscript=no \
+      -drive file=srv${i}/system.img,if=virtio,format=raw \
+      -drive file=srv${i}/seed.img,if=virtio,format=raw \
+      -drive file=srv${i}/data.img,if=virtio,format=raw \
+      -display none -daemonize -pidfile srv${i}/qemu.pid
+  done
+
+  # Give qemu a few seconds to start up networking
+  sleep 5
+}
+
+@test "Wait until Talos API port 50000 is reachable on all machines" {
+  timeout 60 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done'
+}
+
+@test "Generate Talos cluster configuration" {
+  # Cluster‑wide patches
+  cat > patch.yaml <<'EOF'
+machine:
+  kubelet:
+    nodeIP:
+      validSubnets:
+      - 192.168.123.0/24
+    extraConfig:
+      maxPods: 512
+  kernel:
+    modules:
+    - name: openvswitch
+    - name: drbd
+      parameters:
+        - usermode_helper=disabled
+    - name: zfs
+    - name: spl
+  registries:
+    mirrors:
+      docker.io:
+        endpoints:
+        - https://mirror.gcr.io
+  files:
+  - content: |
+      [plugins]
+        [plugins."io.containerd.cri.v1.runtime"]
+          device_ownership_from_security_context = true
+    path: /etc/cri/conf.d/20-customization.part
+    op: create
+
+cluster:
+  apiServer:
+    extraArgs:
+      oidc-issuer-url: "https://keycloak.example.org/realms/cozy"
+      oidc-client-id: "kubernetes"
+      oidc-username-claim: "preferred_username"
+      oidc-groups-claim: "groups"
+  network:
+    cni:
+      name: none
+    dnsDomain: cozy.local
+    podSubnets:
+    - 10.244.0.0/16
+    serviceSubnets:
+    - 10.96.0.0/16
+EOF
+
+  # Control‑plane‑only patches
+  cat > patch-controlplane.yaml <<'EOF'
+machine:
+  nodeLabels:
+    node.kubernetes.io/exclude-from-external-load-balancers:
+      $patch: delete
+  network:
+    interfaces:
+    - interface: eth0
+      vip:
+        ip: 192.168.123.10
+cluster:
+  allowSchedulingOnControlPlanes: true
+  controllerManager:
+    extraArgs:
+      bind-address: 0.0.0.0
+  scheduler:
+    extraArgs:
+      bind-address: 0.0.0.0
+  apiServer:
+    certSANs:
+    - 127.0.0.1
+  proxy:
+    disabled: true
+  discovery:
+    enabled: false
+  etcd:
+    advertisedSubnets:
+    - 192.168.123.0/24
+EOF
+
+  # Generate secrets once
+  if [ ! -f secrets.yaml ]; then
+    talosctl gen secrets
+  fi
+
+  rm -f controlplane.yaml worker.yaml talosconfig kubeconfig
+  talosctl gen config --with-secrets secrets.yaml cozystack https://192.168.123.10:6443 \
+           --config-patch=@patch.yaml --config-patch-control-plane @patch-controlplane.yaml
+}
+
+@test "Apply Talos configuration to the node" {
+  # Apply the configuration to all three nodes
+  for node in 11 12 13; do
+    talosctl apply -f controlplane.yaml -n 192.168.123.${node} -e 192.168.123.${node} -i
+  done
+
+  # Wait for Talos services to come up again
+  timeout 60 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done'
+}
+
+@test "Bootstrap Talos cluster" {
+  # Bootstrap etcd on the first node
+  timeout 10 sh -ec 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 1; done'
+
+  # Wait until etcd is healthy
+  timeout 180 sh -ec 'until talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 >/dev/null 2>&1; do sleep 1; done'
+  timeout 60 sh -ec 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1 | grep -q "rpc error"; do sleep 1; done'
+
+  # Retrieve kubeconfig
+  rm -f kubeconfig
+  talosctl kubeconfig kubeconfig -e 192.168.123.10 -n 192.168.123.10
+
+  # Wait until all three nodes register in Kubernetes
+  timeout 60 sh -ec 'until [ $(kubectl get node --no-headers | wc -l) -eq 3 ]; do sleep 1; done'
+}
+
+@test "Install Cozystack" {
+  # Create namespace & configmap required by installer
+  kubectl create namespace cozy-system --dry-run=client -o yaml | kubectl apply -f -
+  kubectl create configmap cozystack -n cozy-system \
+          --from-literal=bundle-name=paas-full \
+          --from-literal=ipv4-pod-cidr=10.244.0.0/16 \
+          --from-literal=ipv4-pod-gateway=10.244.0.1 \
+          --from-literal=ipv4-svc-cidr=10.96.0.0/16 \
+          --from-literal=ipv4-join-cidr=100.64.0.0/16 \
+          --from-literal=root-host=example.org \
+          --from-literal=api-server-endpoint=https://192.168.123.10:6443 \
+          --dry-run=client -o yaml | kubectl apply -f -
+
+  # Apply installer manifests from file
+  kubectl apply -f _out/assets/cozystack-installer.yaml
+
+  # Wait for the installer deployment to become available
+  kubectl wait deployment/cozystack -n cozy-system --timeout=1m --for=condition=Available
+
+  # Wait until HelmReleases appear & reconcile them
+  timeout 60 sh -ec 'until kubectl get hr -A | grep -q cozys; do sleep 1; done'
+  sleep 5
+  kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n "$1" hr/"$2" &"} END {print "wait"}' | sh -ex
+
+  # Fail the test if any HelmRelease is not Ready
+  if kubectl get hr -A | grep -v " True " | grep -v NAME; then
+    kubectl get hr -A
+    fail "Some HelmReleases failed to reconcile"
+  fi
+}
+
+@test "Wait for Cluster‑API provider deployments" {
+  # Wait for Cluster‑API provider deployments
+  timeout 60 sh -ec 'until kubectl get deploy -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager >/dev/null 2>&1; do sleep 1; done'
+  kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=1m --for=condition=available
+}
+
+@test "Wait for LINSTOR and configure storage" {
+  # Linstor controller and nodes
+  kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available
+  timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done'
+
+  for node in srv1 srv2 srv3; do
+    kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data
+  done
+
+  # Storage classes
+  kubectl apply -f - <<'EOF'
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: local
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: linstor.csi.linbit.com
+parameters:
+  linstor.csi.linbit.com/storagePool: "data"
+  linstor.csi.linbit.com/layerList: "storage"
+  linstor.csi.linbit.com/allowRemoteVolumeAccess: "false"
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: replicated
+provisioner: linstor.csi.linbit.com
+parameters:
+  linstor.csi.linbit.com/storagePool: "data"
+  linstor.csi.linbit.com/autoPlace: "3"
+  linstor.csi.linbit.com/layerList: "drbd storage"
+  linstor.csi.linbit.com/allowRemoteVolumeAccess: "true"
+  property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io
+  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io
+  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary
+  property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+EOF
+}
+
+@test "Wait for MetalLB and configure address pool" {
+  # MetalLB address pool
+  kubectl apply -f - <<'EOF'
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: cozystack
+  namespace: cozy-metallb
+spec:
+  ipAddressPools: [cozystack]
+---
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: cozystack
+  namespace: cozy-metallb
+spec:
+  addresses: [192.168.123.200-192.168.123.250]
+  autoAssign: true
+  avoidBuggyIPs: false
+EOF
+}
+
+@test "Check Cozystack API service" {
+  kubectl wait --for=condition=Available apiservices/v1alpha1.apps.cozystack.io --timeout=2m
+}
+
+@test "Configure Tenant and wait for applications" {
+  # Patch root tenant and wait for its releases
+  kubectl patch tenants/root -n tenant-root --type merge -p '{"spec":{"host":"example.org","ingress":true,"monitoring":true,"etcd":true,"isolated":true}}'
+
+  timeout 60 sh -ec 'until kubectl get hr -n tenant-root etcd ingress monitoring tenant-root >/dev/null 2>&1; do sleep 1; done'
+  kubectl wait hr/etcd hr/ingress hr/tenant-root -n tenant-root --timeout=2m --for=condition=ready
+
+  if ! kubectl wait hr/monitoring -n tenant-root --timeout=2m --for=condition=ready; then
+    flux reconcile hr monitoring -n tenant-root --force
+    kubectl wait hr/monitoring -n tenant-root --timeout=2m --for=condition=ready
+  fi
+
+  # Expose Cozystack services through ingress
+  kubectl patch configmap/cozystack -n cozy-system --type merge -p '{"data":{"expose-services":"api,dashboard,cdi-uploadproxy,vm-exportproxy,keycloak"}}'
+
+  # NGINX ingress controller
+  timeout 60 sh -ec 'until kubectl get deploy root-ingress-controller -n tenant-root >/dev/null 2>&1; do sleep 1; done'
+  kubectl wait deploy/root-ingress-controller -n tenant-root --timeout=5m --for=condition=available
+
+  # etcd statefulset
+  kubectl wait sts/etcd -n tenant-root --for=jsonpath='{.status.readyReplicas}'=3 --timeout=5m
+
+  # VictoriaMetrics components
+  kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m
+  kubectl wait vlogs/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m
+  kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.clusterStatus}'=operational --timeout=5m
+
+  # Grafana
+  kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=5m
+  kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=5m
+
+  # Verify Grafana via ingress
+  ingress_ip=$(kubectl get svc root-ingress-controller -n tenant-root -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  if ! curl -sS -k "https://${ingress_ip}" -H 'Host: grafana.example.org' --max-time 30 | grep -q Found; then
+    echo "Failed to access Grafana via ingress at ${ingress_ip}" >&2
+    exit 1
+  fi
+}
+
+@test "Keycloak OIDC stack is healthy" {
+  kubectl patch configmap/cozystack -n cozy-system --type merge -p '{"data":{"oidc-enabled":"true"}}'
+
+  timeout 120 sh -ec 'until kubectl get hr -n cozy-keycloak keycloak keycloak-configure keycloak-operator >/dev/null 2>&1; do sleep 1; done'
+  kubectl wait hr/keycloak hr/keycloak-configure hr/keycloak-operator -n cozy-keycloak --timeout=10m --for=condition=ready
+}

hack/e2e.application.sh

@@ -1,165 +0,0 @@
-#!/bin/bash
-
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-RESET='\033[0m'
-YELLOW='\033[0;33m'
-
-
-ROOT_NS="tenant-root"
-TEST_TENANT="tenant-e2e"
-
-values_base_path="/hack/testdata/"
-checks_base_path="/hack/testdata/"
-
-function delete_hr() {
-    local release_name="$1"
-    local namespace="$2"
-
-    if [[ -z "$release_name" ]]; then
-        echo -e "${RED}Error: Release name is required.${RESET}"
-        exit 1
-    fi
-
-    if [[ -z "$namespace" ]]; then
-        echo -e "${RED}Error: Namespace name is required.${RESET}"
-        exit 1
-    fi
-
-    if [[ "$release_name" == "tenant-e2e" ]]; then
-        echo -e "${YELLOW}Skipping deletion for release tenant-e2e.${RESET}"
-        return 0
-    fi
-
-    kubectl delete helmrelease $release_name -n $namespace
-}
-
-function install_helmrelease() {
-    local release_name="$1"
-    local namespace="$2"
-    local chart_path="$3"
-    local repo_name="$4"
-    local repo_ns="$5"
-    local values_file="$6"
-
-    if [[ -z "$release_name" ]]; then
-        echo -e "${RED}Error: Release name is required.${RESET}"
-        exit 1
-    fi
-
-    if [[ -z "$namespace" ]]; then
-        echo -e "${RED}Error: Namespace name is required.${RESET}"
-        exit 1
-    fi
-
-    if [[ -z "$chart_path" ]]; then
-        echo -e "${RED}Error: Chart path name is required.${RESET}"
-        exit 1
-    fi
-
-    if [[ -n "$values_file" && -f "$values_file" ]]; then
-        local values_section
-        values_section=$(echo "  values:" && sed 's/^/    /' "$values_file")
-    fi
-
-    local helmrelease_file=$(mktemp /tmp/HelmRelease.XXXXXX.yaml)
-    {
-        echo "apiVersion: helm.toolkit.fluxcd.io/v2"
-        echo "kind: HelmRelease"
-        echo "metadata:"
-        echo "  labels:"
-        echo "    cozystack.io/ui: \"true\""
-        echo "  name: \"$release_name\""
-        echo "  namespace: \"$namespace\""
-        echo "spec:"
-        echo "  chart:"
-        echo "    spec:"
-        echo "      chart: \"$chart_path\""
-        echo "      reconcileStrategy: Revision"
-        echo "      sourceRef:"
-        echo "        kind: HelmRepository"
-        echo "        name: \"$repo_name\""
-        echo "        namespace: \"$repo_ns\""
-        echo "      version: '*'"
-        echo "  interval: 1m0s"
-        echo "  timeout: 5m0s"
-        [[ -n "$values_section" ]] && echo "$values_section"
-    } > "$helmrelease_file"
-
-    kubectl apply -f "$helmrelease_file"
-
-    rm -f "$helmrelease_file"
-}
-
-function install_tenant (){
-    local release_name="$1"
-    local namespace="$2"
-    local values_file="${values_base_path}tenant/values.yaml"
-    local repo_name="cozystack-apps"
-    local repo_ns="cozy-public"
-    install_helmrelease "$release_name" "$namespace" "tenant" "$repo_name" "$repo_ns" "$values_file"
-}
-
-function make_extra_checks(){
-    local checks_file="$1"
-    echo "after exec make $checks_file"
-    if [[ -n "$checks_file" && -f "$checks_file" ]]; then
-        echo -e "${YELLOW}Start extra checks with file: ${checks_file}${RESET}"
-
-    fi
-}
-
-function check_helmrelease_status() {
-    local release_name="$1"
-    local namespace="$2"
-    local checks_file="$3"
-    local timeout=300  # Timeout in seconds
-    local interval=5   # Interval between checks in seconds
-    local elapsed=0
-
-
-    while [[ $elapsed -lt $timeout ]]; do
-        local status_output
-        status_output=$(kubectl get helmrelease "$release_name" -n "$namespace" -o json | jq -r '.status.conditions[-1].reason')
-
-        if [[ "$status_output" == "InstallSucceeded" || "$status_output" == "UpgradeSucceeded" ]]; then
-            echo -e "${GREEN}Helm release '$release_name' is ready.${RESET}"
-            make_extra_checks "$checks_file"
-            delete_hr $release_name $namespace
-            return 0
-        elif [[ "$status_output" == "InstallFailed" ]]; then
-          echo -e "${RED}Helm release '$release_name': InstallFailed${RESET}"
-          exit 1
-        else
-            echo -e "${YELLOW}Helm release '$release_name' is not ready. Current status: $status_output${RESET}"
-        fi
-
-        sleep "$interval"
-        elapsed=$((elapsed + interval))
-    done
-
-    echo -e "${RED}Timeout reached. Helm release '$release_name' is still not ready after $timeout seconds.${RESET}"
-    exit 1
-}
-
-chart_name="$1"
-
-if [ -z "$chart_name" ]; then
-    echo -e "${RED}No chart name provided. Exiting...${RESET}"
-    exit 1
-fi
-
-
-checks_file="${checks_base_path}${chart_name}/check.sh"
-repo_name="cozystack-apps"
-repo_ns="cozy-public"
-release_name="$chart_name-e2e"
-values_file="${values_base_path}${chart_name}/values.yaml"
-
-install_tenant $TEST_TENANT $ROOT_NS
-check_helmrelease_status $TEST_TENANT $ROOT_NS "${checks_base_path}tenant/check.sh"
-
-echo -e "${YELLOW}Running tests for chart: $chart_name${RESET}"
-
-install_helmrelease $release_name $TEST_TENANT $chart_name $repo_name $repo_ns $values_file
-check_helmrelease_status $release_name $TEST_TENANT $checks_file

hack/e2e.sh

@@ -1,351 +0,0 @@
-#!/bin/bash
-if [ "$COZYSTACK_INSTALLER_YAML" = "" ]; then
-  echo 'COZYSTACK_INSTALLER_YAML variable is not set!' >&2
-  echo 'please set it with following command:' >&2
-  echo >&2
-  echo 'export COZYSTACK_INSTALLER_YAML=$(helm template -n cozy-system installer packages/core/installer)' >&2
-  echo >&2
-  exit 1
-fi
-
-if [ "$(cat /proc/sys/net/ipv4/ip_forward)" != 1 ]; then
-  echo "IPv4 forwarding is not enabled!" >&2
-  echo 'please enable forwarding with the following command:' >&2
-  echo >&2
-  echo 'echo 1 > /proc/sys/net/ipv4/ip_forward' >&2
-  echo >&2
-  exit 1
-fi
-
-set -x
-set -e
-
-kill `cat srv1/qemu.pid srv2/qemu.pid srv3/qemu.pid` || true
-
-ip link del cozy-br0 || true
-ip link add cozy-br0 type bridge
-ip link set cozy-br0 up
-ip addr add 192.168.123.1/24 dev cozy-br0
-
-# Enable masquerading
-iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE 2>/dev/null || true
-iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE
-
-rm -rf srv1 srv2 srv3
-mkdir -p srv1 srv2 srv3
-
-# Prepare cloud-init
-for i in 1 2 3; do
-  echo "hostname: srv$i" > "srv$i/meta-data"
-  echo '#cloud-config' > "srv$i/user-data"
-  cat > "srv$i/network-config" <<EOT
-version: 2
-ethernets:
-  eth0:
-    dhcp4: false
-    addresses:
-      - "192.168.123.1$i/26"
-    gateway4: "192.168.123.1"
-    nameservers:
-      search: [cluster.local]
-      addresses: [8.8.8.8]
-EOT
-
-  ( cd srv$i && genisoimage \
-      -output seed.img \
-      -volid cidata -rational-rock -joliet \
-      user-data meta-data network-config
-  )
-done
-
-# Prepare system drive
-if [ ! -f nocloud-amd64.raw ]; then
-  wget https://github.com/cozystack/cozystack/releases/latest/download/nocloud-amd64.raw.xz -O nocloud-amd64.raw.xz
-  rm -f nocloud-amd64.raw
-  xz --decompress nocloud-amd64.raw.xz
-fi
-for i in 1 2 3; do
-  cp nocloud-amd64.raw srv$i/system.img
-  qemu-img resize srv$i/system.img 20G
-done
-
-# Prepare data drives
-for i in 1 2 3; do
-  qemu-img create srv$i/data.img 100G
-done
-
-# Prepare networking
-for i in 1 2 3; do
-  ip link del cozy-srv$i || true
-  ip tuntap add dev cozy-srv$i mode tap
-  ip link set cozy-srv$i up
-  ip link set cozy-srv$i master cozy-br0
-done
-
-# Start VMs
-for i in 1 2 3; do
-  qemu-system-x86_64 -machine type=pc,accel=kvm -cpu host -smp 4 -m 8192 \
-    -device virtio-net,netdev=net0,mac=52:54:00:12:34:5$i -netdev tap,id=net0,ifname=cozy-srv$i,script=no,downscript=no \
-    -drive file=srv$i/system.img,if=virtio,format=raw \
-    -drive file=srv$i/seed.img,if=virtio,format=raw \
-    -drive file=srv$i/data.img,if=virtio,format=raw \
-    -display none -daemonize -pidfile srv$i/qemu.pid
-done
-
-sleep 5
-
-# Wait for VM to start up
-timeout 60 sh -c 'until nc -nzv 192.168.123.11 50000 && nc -nzv 192.168.123.12 50000 && nc -nzv 192.168.123.13 50000; do sleep 1; done'
-
-cat > patch.yaml <<\EOT
-machine:
-  kubelet:
-    nodeIP:
-      validSubnets:
-      - 192.168.123.0/24
-    extraConfig:
-      maxPods: 512
-  kernel:
-    modules:
-    - name: openvswitch
-    - name: drbd
-      parameters:
-        - usermode_helper=disabled
-    - name: zfs
-    - name: spl
-  files:
-  - content: |
-      [plugins]
-        [plugins."io.containerd.grpc.v1.cri"]
-          device_ownership_from_security_context = true      
-    path: /etc/cri/conf.d/20-customization.part
-    op: create
-
-cluster:
-  apiServer:
-    extraArgs:
-      oidc-issuer-url: "https://keycloak.example.org/realms/cozy"
-      oidc-client-id: "kubernetes"
-      oidc-username-claim: "preferred_username"
-      oidc-groups-claim: "groups"
-  network:
-    cni:
-      name: none
-    dnsDomain: cozy.local
-    podSubnets:
-    - 10.244.0.0/16
-    serviceSubnets:
-    - 10.96.0.0/16
-EOT
-
-cat > patch-controlplane.yaml <<\EOT
-machine:
-  nodeLabels:
-    node.kubernetes.io/exclude-from-external-load-balancers:
-      $patch: delete
-  network:
-    interfaces:
-    - interface: eth0
-      vip:
-        ip: 192.168.123.10
-cluster:
-  allowSchedulingOnControlPlanes: true
-  controllerManager:
-    extraArgs:
-      bind-address: 0.0.0.0
-  scheduler:
-    extraArgs:
-      bind-address: 0.0.0.0
-  apiServer:
-    certSANs:
-    - 127.0.0.1
-  proxy:
-    disabled: true
-  discovery:
-    enabled: false
-  etcd:
-    advertisedSubnets:
-    - 192.168.123.0/24
-EOT
-
-# Gen configuration
-if [ ! -f secrets.yaml ]; then
-  talosctl gen secrets
-fi
-
-rm -f controlplane.yaml worker.yaml talosconfig kubeconfig
-talosctl gen config --with-secrets secrets.yaml cozystack https://192.168.123.10:6443 --config-patch=@patch.yaml --config-patch-control-plane @patch-controlplane.yaml
-export TALOSCONFIG=$PWD/talosconfig
-
-# Apply configuration
-talosctl apply -f controlplane.yaml -n 192.168.123.11 -e 192.168.123.11 -i
-talosctl apply -f controlplane.yaml -n 192.168.123.12 -e 192.168.123.12 -i
-talosctl apply -f controlplane.yaml -n 192.168.123.13 -e 192.168.123.13 -i
-
-# Wait for VM to be configured
-timeout 60 sh -c 'until nc -nzv 192.168.123.11 50000 && nc -nzv 192.168.123.12 50000 && nc -nzv 192.168.123.13 50000; do sleep 1; done'
-
-# Bootstrap
-timeout 10 sh -c 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 1; done'
-
-# Wait for etcd
-timeout 180 sh -c 'until timeout -s 9 2 talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1; do sleep 1; done'
-timeout 60 sh -c 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1 | grep "rpc error"; do sleep 1; done'
-
-rm -f kubeconfig
-talosctl kubeconfig kubeconfig -e 192.168.123.10 -n 192.168.123.10
-export KUBECONFIG=$PWD/kubeconfig
-
-# Wait for kubernetes nodes appear
-timeout 60 sh -c 'until [ $(kubectl get node -o name | wc -l) = 3 ]; do sleep 1; done'
-kubectl create ns cozy-system -o yaml | kubectl apply -f -
-kubectl create -f - <<\EOT
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: cozystack
-  namespace: cozy-system
-data:
-  bundle-name: "paas-full"
-  ipv4-pod-cidr: "10.244.0.0/16"
-  ipv4-pod-gateway: "10.244.0.1"
-  ipv4-svc-cidr: "10.96.0.0/16"
-  ipv4-join-cidr: "100.64.0.0/16"
-  root-host: example.org
-  api-server-endpoint: https://192.168.123.10:6443
-EOT
-
-#
-echo "$COZYSTACK_INSTALLER_YAML" | kubectl apply -f -
-
-# wait for cozystack pod to start
-kubectl wait deploy  --timeout=1m --for=condition=available -n cozy-system cozystack
-
-# wait for helmreleases appear
-timeout 60 sh -c 'until kubectl get hr -A | grep cozy; do sleep 1; done'
-
-sleep 5
-
-kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n " $1 " hr/" $2 " &"} END{print "wait"}' | sh -x
-
-# Wait for Cluster-API providers
-timeout 30 sh -c 'until kubectl get deploy -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager; do sleep 1; done'
-kubectl wait deploy --timeout=30s --for=condition=available -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager
-
-# Wait for linstor controller
-kubectl wait deploy --timeout=5m --for=condition=available -n cozy-linstor linstor-controller
-
-# Wait for all linstor nodes become Online
-timeout 60 sh -c 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) = 3 ]; do sleep 1; done'
-
-kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs srv1 /dev/vdc --pool-name data --storage-pool data
-kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs srv2 /dev/vdc --pool-name data --storage-pool data
-kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs srv3 /dev/vdc --pool-name data --storage-pool data
-
-kubectl create -f- <<EOT
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: local
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-provisioner: linstor.csi.linbit.com
-parameters:
-  linstor.csi.linbit.com/storagePool: "data"
-  linstor.csi.linbit.com/layerList: "storage"
-  linstor.csi.linbit.com/allowRemoteVolumeAccess: "false"
-volumeBindingMode: WaitForFirstConsumer
-allowVolumeExpansion: true
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: replicated
-provisioner: linstor.csi.linbit.com
-parameters:
-  linstor.csi.linbit.com/storagePool: "data"
-  linstor.csi.linbit.com/autoPlace: "3"
-  linstor.csi.linbit.com/layerList: "drbd storage"
-  linstor.csi.linbit.com/allowRemoteVolumeAccess: "true"
-  property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io
-  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io
-  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary
-  property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect
-volumeBindingMode: WaitForFirstConsumer
-allowVolumeExpansion: true
-EOT
-kubectl create -f- <<EOT
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: cozystack
-  namespace: cozy-metallb
-spec:
-  ipAddressPools:
-  - cozystack
----
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
-  name: cozystack
-  namespace: cozy-metallb
-spec:
-  addresses:
-  - 192.168.123.200-192.168.123.250
-  autoAssign: true
-  avoidBuggyIPs: false
-EOT
-
-# Wait for cozystack-api
-kubectl wait --for=condition=Available apiservices v1alpha1.apps.cozystack.io --timeout=2m
-
-kubectl patch -n tenant-root tenants.apps.cozystack.io root --type=merge -p '{"spec":{
-  "host": "example.org",
-  "ingress": true,
-  "monitoring": true,
-  "etcd": true,
-  "isolated": true
-}}'
-
-# Wait for HelmRelease be created
-timeout 60 sh -c 'until kubectl get hr -n tenant-root etcd ingress monitoring tenant-root; do sleep 1; done'
-
-# Wait for HelmReleases be installed
-kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr etcd ingress monitoring tenant-root
-
-kubectl patch -n tenant-root ingresses.apps.cozystack.io ingress --type=merge -p '{"spec":{
-  "dashboard": true
-}}'
-
-# Wait for nginx-ingress-controller
-timeout 60 sh -c 'until kubectl get deploy -n tenant-root root-ingress-controller; do sleep 1; done'
-kubectl wait --timeout=5m --for=condition=available -n tenant-root deploy root-ingress-controller
-
-# Wait for etcd
-kubectl wait --timeout=5m --for=jsonpath=.status.readyReplicas=3 -n tenant-root sts etcd
-
-# Wait for Victoria metrics
-kubectl wait --timeout=5m --for=jsonpath=.status.updateStatus=operational -n tenant-root vmalert/vmalert-shortterm vmalertmanager/alertmanager
-kubectl wait --timeout=5m --for=jsonpath=.status.status=operational -n tenant-root vlogs/generic
-kubectl wait --timeout=5m --for=jsonpath=.status.clusterStatus=operational -n tenant-root vmcluster/shortterm vmcluster/longterm
-
-# Wait for grafana
-kubectl wait --timeout=5m --for=condition=ready -n tenant-root clusters.postgresql.cnpg.io grafana-db
-kubectl wait --timeout=5m --for=condition=available -n tenant-root deploy grafana-deployment 
-
-# Get IP of nginx-ingress
-ip=$(kubectl get svc -n tenant-root root-ingress-controller -o jsonpath='{.status.loadBalancer.ingress..ip}')
-
-# Check Grafana
-curl -sS -k "https://$ip" -H 'Host: grafana.example.org' | grep Found
-
-
-# Test OIDC
-kubectl patch -n cozy-system cm/cozystack --type=merge -p '{"data":{
-  "oidc-enabled": "true"
-}}'
-
-timeout 60 sh -c 'until kubectl get hr -n cozy-keycloak keycloak keycloak-configure keycloak-operator; do sleep 1; done'
-kubectl wait --timeout=10m --for=condition=ready -n cozy-keycloak hr keycloak keycloak-configure keycloak-operator

hack/gen_versions_map.sh

@@ -1,12 +1,13 @@
 #!/bin/sh
 set -e
+
 file=versions_map
+
 charts=$(find . -mindepth 2 -maxdepth 2 -name Chart.yaml | awk 'sub("/Chart.yaml", "")')
 
-# <chart> <version> <commit> 
 new_map=$(
   for chart in $charts; do
-    awk '/^name:/ {chart=$2} /^version:/ {version=$2} END{printf "%s %s %s\n", chart, version, "HEAD"}' $chart/Chart.yaml
+    awk '/^name:/ {chart=$2} /^version:/ {version=$2} END{printf "%s %s %s\n", chart, version, "HEAD"}' "$chart/Chart.yaml"
   done
 )
 
@@ -15,48 +16,49 @@ if [ ! -f "$file" ] || [ ! -s "$file" ]; then
   exit 0
 fi
 
-miss_map=$(echo "$new_map" | awk 'NR==FNR { new_map[$1 " " $2] = $3; next } { if (!($1 " " $2 in new_map)) print $1, $2, $3}' - $file)
+miss_map=$(mktemp)
+trap 'rm -f "$miss_map"' EXIT
+echo -n "$new_map" | awk 'NR==FNR { nm[$1 " " $2] = $3; next } { if (!($1 " " $2 in nm)) print $1, $2, $3}' - "$file" > $miss_map
+
+# search accross all tags sorted by version
+search_commits=$(git ls-remote --tags origin | awk -F/ '$3 ~ /v[0-9]+.[0-9]+.[0-9]+/ {print}' | sort -k2,2 -rV | awk '{print $1}')
 
 resolved_miss_map=$(
-  echo "$miss_map" | while read chart version commit; do
-    if [ "$commit" = HEAD ]; then
-      line=$(awk '/^version:/ {print NR; exit}' "./$chart/Chart.yaml")
-      change_commit=$(git --no-pager blame -L"$line",+1 -- "$chart/Chart.yaml" | awk '{print $1}')
-       
-      if [ "$change_commit" = "00000000" ]; then
-        # Not committed yet, use previous commit
-        line=$(git show HEAD:"./$chart/Chart.yaml" | awk '/^version:/ {print NR; exit}')
-        commit=$(git --no-pager blame -L"$line",+1 HEAD -- "$chart/Chart.yaml" | awk '{print $1}')
-        if [ $(echo $commit | cut -c1) = "^" ]; then
-          # Previous commit not exists
-          commit=$(echo $commit | cut -c2-)
-        fi
-      else
-        # Committed, but version_map wasn't updated
-        line=$(git show HEAD:"./$chart/Chart.yaml" | awk '/^version:/ {print NR; exit}')
-        change_commit=$(git --no-pager blame -L"$line",+1 HEAD -- "$chart/Chart.yaml" | awk '{print $1}')
-        if [ $(echo $change_commit | cut -c1) = "^" ]; then
-          # Previous commit not exists
-          commit=$(echo $change_commit | cut -c2-)
-        else
-          commit=$(git describe --always "$change_commit~1")
-        fi
+  while read -r chart version commit; do
+    # if version is found in HEAD, it's HEAD
+    if [ "$(awk '$1 == "version:" {print $2}' ./${chart}/Chart.yaml)" = "${version}" ]; then
+      echo "$chart $version HEAD"
+      continue
+    fi
+
+    # if commit is not HEAD, check if it's valid
+    if [ "$commit" != "HEAD" ]; then
+      if [ "$(git show "${commit}:./${chart}/Chart.yaml" | awk '$1 == "version:" {print $2}')" != "${version}" ]; then
+        echo "Commit $commit for $chart $version is not valid" >&2
+        exit 1
       fi
 
-      # Check if the commit belongs to the main branch
-      if ! git merge-base --is-ancestor "$commit" main; then
-        # Find the closest parent commit that belongs to main
-        commit_in_main=$(git log --pretty=format:"%h" main -- "$chart" | head -n 1)
-        if [ -n "$commit_in_main" ]; then
-          commit="$commit_in_main"
-        else
-          # No valid commit found in main branch for $chart, skipping..."
-          continue
-        fi
-      fi
+      commit=$(git rev-parse --short "$commit")
+      echo "$chart $version $commit"
+      continue
     fi
-    echo "$chart $version $commit"
-  done
+
+    # if commit is HEAD, but version is not found in HEAD, check all tags
+    found_tag=""
+    for tag in $search_commits; do
+      if [ "$(git show "${tag}:./${chart}/Chart.yaml" | awk '$1 == "version:" {print $2}')" = "${version}" ]; then
+        found_tag=$(git rev-parse --short "${tag}")
+        break
+      fi
+    done
+    
+    if [ -z "$found_tag" ]; then
+      echo "Can't find $chart $version in any version tag, removing it" >&2
+      continue
+    fi
+    
+    echo "$chart $version $found_tag"
+  done < $miss_map
 )
 
 printf "%s\n" "$new_map" "$resolved_miss_map" | sort -k1,1 -k2,2 -V | awk '$1' > "$file"

hack/package_chart.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+set -e
+
+usage() {
+        printf "%s\n" "Usage:" >&2 ;
+        printf -- "%s\n" '---' >&2 ;
+        printf "%s %s\n" "$0" "INPUT_DIR OUTPUT_DIR TMP_DIR [DEPENDENCY_DIR]" >&2 ;
+        printf -- "%s\n" '---' >&2 ;
+        printf "%s\n" "Takes a helm repository from INPUT_DIR, with an optional library repository in" >&2 ;
+        printf "%s\n" "DEPENDENCY_DIR, prepares a view of the git archive at select points in history" >&2 ;
+        printf "%s\n" "in TMP_DIR and packages helm charts, outputting the tarballs to OUTPUT_DIR" >&2 ;
+}
+
+if [ "x$(basename $PWD)" != "xpackages" ]
+then
+        echo "Error: This script must run from the ./packages/ directory" >&2
+        echo >&2
+        usage
+        exit 1
+fi
+
+if [ "x$#" != "x3" ] && [ "x$#" != "x4" ]
+then
+        echo "Error: This script takes 3 or 4 arguments" >&2
+        echo "Got $# arguments:" "$@" >&2
+        echo >&2
+        usage
+        exit 1
+fi
+
+input_dir=$1
+output_dir=$2
+tmp_dir=$3
+
+if [ "x$#" = "x4" ]
+then
+        dependency_dir=$4
+fi
+
+rm -rf "${output_dir:?}"
+mkdir -p "${output_dir}"
+while read package _ commit
+do
+        # this lets devs build the packages from a dirty repo for quick local testing
+        if [ "x$commit" = "xHEAD" ]
+        then
+                helm package "${input_dir}/${package}" -d "${output_dir}"
+                continue
+        fi
+        git archive --format tar "${commit}" "${input_dir}/${package}" | tar -xf- -C "${tmp_dir}/"
+
+        # the library chart is not present in older commits and git archive doesn't fail gracefully if the path is not found
+        if [ "x${dependency_dir}" != "x" ] && git ls-tree --name-only "${commit}" "${dependency_dir}" | grep -qx "${dependency_dir}"
+        then
+                git archive --format tar "${commit}" "${dependency_dir}" | tar -xf- -C "${tmp_dir}/"
+        fi
+        helm package "${tmp_dir}/${input_dir}/${package}" -d "${output_dir}"
+        rm -rf "${tmp_dir:?}/${input_dir:?}/${package:?}"
+        if [ "x${dependency_dir}" != "x" ]
+        then
+                rm -rf "${tmp_dir:?}/${dependency_dir:?}"
+        fi
+done < "${input_dir}/versions_map"
+helm repo index "${output_dir}"

hack/testdata/http-cache/check.sh

@@ -1 +0,0 @@
-return 0

hack/testdata/http-cache/values.yaml

@@ -1,2 +0,0 @@
-endpoints:
-  - 8.8.8.8:443

hack/testdata/kubernetes/check.sh

@@ -1 +0,0 @@
-return 0

hack/testdata/kubernetes/values.yaml

@@ -1,62 +0,0 @@
-## @section Common parameters
-
-## @param host The hostname used to access the Kubernetes cluster externally (defaults to using the cluster name as a subdomain for the tenant host).
-## @param controlPlane.replicas Number of replicas for Kubernetes contorl-plane components
-## @param storageClass StorageClass used to store user data
-##
-host: ""
-controlPlane:
-  replicas: 2
-storageClass: replicated
-
-## @param nodeGroups [object] nodeGroups configuration
-##
-nodeGroups:
-  md0:
-    minReplicas: 0
-    maxReplicas: 10
-    instanceType: "u1.medium"
-    ephemeralStorage: 20Gi
-    roles:
-    - ingress-nginx
-
-    resources:
-      cpu: ""
-      memory: ""
-
-## @section Cluster Addons
-##
-addons:
-
-  ## Cert-manager: automatically creates and manages SSL/TLS certificate
-  ##
-  certManager:
-    ## @param addons.certManager.enabled Enables the cert-manager
-    ## @param addons.certManager.valuesOverride Custom values to override
-    enabled: true
-    valuesOverride: {}
-
-  ## Ingress-NGINX Controller
-  ##
-  ingressNginx:
-    ## @param addons.ingressNginx.enabled Enable Ingress-NGINX controller (expect nodes with 'ingress-nginx' role)
-    ## @param addons.ingressNginx.valuesOverride Custom values to override
-    ##
-    enabled: true
-    ## @param addons.ingressNginx.hosts List of domain names that should be passed through to the cluster by upper cluster
-    ## e.g:
-    ## hosts:
-    ## - example.org
-    ## - foo.example.net
-    ##
-    hosts: []
-    valuesOverride: {}
-
-  ## Flux CD
-  ##
-  fluxcd:
-    ## @param addons.fluxcd.enabled Enables Flux CD
-    ## @param addons.fluxcd.valuesOverride Custom values to override
-    ##
-    enabled: true
-    valuesOverride: {}

hack/testdata/nats/check.sh

@@ -1 +0,0 @@
-return 0

hack/testdata/nats/values.yaml

@@ -1,10 +0,0 @@
-
-## @section Common parameters
-
-## @param external Enable external access from outside the cluster
-## @param replicas Persistent Volume size for NATS
-## @param storageClass StorageClass used to store the data
-##
-external: false
-replicas: 2
-storageClass: ""

hack/testdata/tenant/check.sh

@@ -1 +0,0 @@
-return 0

hack/testdata/tenant/values.yaml

@@ -1,6 +0,0 @@
-host: ""
-etcd: false
-monitoring: false
-ingress: false
-seaweedfs: false
-isolated: true

hack/upload-assets.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -xe
+
+version=${VERSION:-$(git describe --tags)}
+
+gh release upload --clobber $version _out/assets/cozystack-installer.yaml
+gh release upload --clobber $version _out/assets/metal-amd64.iso
+gh release upload --clobber $version _out/assets/metal-amd64.raw.xz
+gh release upload --clobber $version _out/assets/nocloud-amd64.raw.xz
+gh release upload --clobber $version _out/assets/kernel-amd64
+gh release upload --clobber $version _out/assets/initramfs-metal-amd64.xz

internal/controller/system_helm_reconciler.go

@@ -0,0 +1,139 @@
+package controller
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"time"
+
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+type CozystackConfigReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+}
+
+var configMapNames = []string{"cozystack", "cozystack-branding", "cozystack-scheduling"}
+
+const configMapNamespace = "cozy-system"
+const digestAnnotation = "cozystack.io/cozy-config-digest"
+const forceReconcileKey = "reconcile.fluxcd.io/forceAt"
+const requestedAt = "reconcile.fluxcd.io/requestedAt"
+
+func (r *CozystackConfigReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) {
+	log := log.FromContext(ctx)
+
+	digest, err := r.computeDigest(ctx)
+	if err != nil {
+		log.Error(err, "failed to compute config digest")
+		return ctrl.Result{}, nil
+	}
+
+	var helmList helmv2.HelmReleaseList
+	if err := r.List(ctx, &helmList); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to list HelmReleases: %w", err)
+	}
+
+	now := time.Now().Format(time.RFC3339Nano)
+	updated := 0
+
+	for _, hr := range helmList.Items {
+		isSystemApp := hr.Labels["cozystack.io/system-app"] == "true"
+		isTenantRoot := hr.Namespace == "tenant-root" && hr.Name == "tenant-root"
+		if !isSystemApp && !isTenantRoot {
+			continue
+		}
+
+		if hr.Annotations == nil {
+			hr.Annotations = map[string]string{}
+		}
+
+		if hr.Annotations[digestAnnotation] == digest {
+			continue
+		}
+
+		patch := client.MergeFrom(hr.DeepCopy())
+		hr.Annotations[digestAnnotation] = digest
+		hr.Annotations[forceReconcileKey] = now
+		hr.Annotations[requestedAt] = now
+
+		if err := r.Patch(ctx, &hr, patch); err != nil {
+			log.Error(err, "failed to patch HelmRelease", "name", hr.Name, "namespace", hr.Namespace)
+			continue
+		}
+		updated++
+		log.Info("patched HelmRelease with new config digest", "name", hr.Name, "namespace", hr.Namespace)
+	}
+
+	log.Info("finished reconciliation", "updatedHelmReleases", updated)
+	return ctrl.Result{}, nil
+}
+
+func (r *CozystackConfigReconciler) computeDigest(ctx context.Context) (string, error) {
+	hash := sha256.New()
+
+	for _, name := range configMapNames {
+		var cm corev1.ConfigMap
+		err := r.Get(ctx, client.ObjectKey{Namespace: configMapNamespace, Name: name}, &cm)
+		if err != nil {
+			if kerrors.IsNotFound(err) {
+				continue // ignore missing
+			}
+			return "", err
+		}
+
+		// Sort keys for consistent hashing
+		var keys []string
+		for k := range cm.Data {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		for _, k := range keys {
+			v := cm.Data[k]
+			fmt.Fprintf(hash, "%s:%s=%s\n", name, k, v)
+		}
+	}
+
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+func (r *CozystackConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		WithEventFilter(predicate.Funcs{
+			UpdateFunc: func(e event.UpdateEvent) bool {
+				cm, ok := e.ObjectNew.(*corev1.ConfigMap)
+				return ok && cm.Namespace == configMapNamespace && contains(configMapNames, cm.Name)
+			},
+			CreateFunc: func(e event.CreateEvent) bool {
+				cm, ok := e.Object.(*corev1.ConfigMap)
+				return ok && cm.Namespace == configMapNamespace && contains(configMapNames, cm.Name)
+			},
+			DeleteFunc: func(e event.DeleteEvent) bool {
+				cm, ok := e.Object.(*corev1.ConfigMap)
+				return ok && cm.Namespace == configMapNamespace && contains(configMapNames, cm.Name)
+			},
+		}).
+		For(&corev1.ConfigMap{}).
+		Complete(r)
+}
+
+func contains(slice []string, val string) bool {
+	for _, s := range slice {
+		if s == val {
+			return true
+		}
+	}
+	return false
+}

internal/controller/tenant_helm_reconciler.go

@@ -0,0 +1,158 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	e "errors"
+
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	"gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+type TenantHelmReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+}
+
+func (r *TenantHelmReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+
+	hr := &helmv2.HelmRelease{}
+	if err := r.Get(ctx, req.NamespacedName, hr); err != nil {
+		if errors.IsNotFound(err) {
+			return ctrl.Result{}, nil
+		}
+		logger.Error(err, "unable to fetch HelmRelease")
+		return ctrl.Result{}, err
+	}
+
+	if !strings.HasPrefix(hr.Name, "tenant-") {
+		return ctrl.Result{}, nil
+	}
+
+	if len(hr.Status.Conditions) == 0 || hr.Status.Conditions[0].Type != "Ready" {
+		return ctrl.Result{}, nil
+	}
+
+	if len(hr.Status.History) == 0 {
+		logger.Info("no history in HelmRelease status", "name", hr.Name)
+		return ctrl.Result{}, nil
+	}
+
+	if hr.Status.History[0].Status != "deployed" {
+		return ctrl.Result{}, nil
+	}
+
+	newDigest := hr.Status.History[0].Digest
+	var hrList helmv2.HelmReleaseList
+	childNamespace := getChildNamespace(hr.Namespace, hr.Name)
+	if childNamespace == "tenant-root" && hr.Name == "tenant-root" {
+		if hr.Spec.Values == nil {
+			logger.Error(e.New("hr.Spec.Values is nil"), "cant annotate tenant-root ns")
+			return ctrl.Result{}, nil
+		}
+		err := annotateTenantRootNs(*hr.Spec.Values, r.Client)
+		if err != nil {
+			logger.Error(err, "cant annotate tenant-root ns")
+			return ctrl.Result{}, nil
+		}
+		logger.Info("namespace 'tenant-root' annotated")
+	}
+
+	if err := r.List(ctx, &hrList, client.InNamespace(childNamespace)); err != nil {
+		logger.Error(err, "unable to list HelmReleases in namespace", "namespace", hr.Name)
+		return ctrl.Result{}, err
+	}
+
+	for _, item := range hrList.Items {
+		if item.Name == hr.Name {
+			continue
+		}
+		oldDigest := item.GetAnnotations()["cozystack.io/tenant-config-digest"]
+		if oldDigest == newDigest {
+			continue
+		}
+		patchTarget := item.DeepCopy()
+
+		if patchTarget.Annotations == nil {
+			patchTarget.Annotations = map[string]string{}
+		}
+		ts := time.Now().Format(time.RFC3339Nano)
+
+		patchTarget.Annotations["cozystack.io/tenant-config-digest"] = newDigest
+		patchTarget.Annotations["reconcile.fluxcd.io/forceAt"] = ts
+		patchTarget.Annotations["reconcile.fluxcd.io/requestedAt"] = ts
+
+		patch := client.MergeFrom(item.DeepCopy())
+		if err := r.Patch(ctx, patchTarget, patch); err != nil {
+			logger.Error(err, "failed to patch HelmRelease", "name", patchTarget.Name)
+			continue
+		}
+
+		logger.Info("patched HelmRelease with new digest", "name", patchTarget.Name, "digest", newDigest, "version", hr.Status.History[0].Version)
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *TenantHelmReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&helmv2.HelmRelease{}).
+		Complete(r)
+}
+
+func getChildNamespace(currentNamespace, hrName string) string {
+	tenantName := strings.TrimPrefix(hrName, "tenant-")
+
+	switch {
+	case currentNamespace == "tenant-root" && hrName == "tenant-root":
+		// 1) root tenant inside root namespace
+		return "tenant-root"
+
+	case currentNamespace == "tenant-root":
+		// 2) any other tenant in root namespace
+		return fmt.Sprintf("tenant-%s", tenantName)
+
+	default:
+		// 3) tenant in a dedicated namespace
+		return fmt.Sprintf("%s-%s", currentNamespace, tenantName)
+	}
+}
+
+func annotateTenantRootNs(values apiextensionsv1.JSON, c client.Client) error {
+	var data map[string]interface{}
+	if err := yaml.Unmarshal(values.Raw, &data); err != nil {
+		return fmt.Errorf("failed to parse HelmRelease values: %w", err)
+	}
+
+	host, ok := data["host"].(string)
+	if !ok || host == "" {
+		return fmt.Errorf("host field not found or not a string")
+	}
+
+	var ns corev1.Namespace
+	if err := c.Get(context.TODO(), client.ObjectKey{Name: "tenant-root"}, &ns); err != nil {
+		return fmt.Errorf("failed to get namespace tenant-root: %w", err)
+	}
+
+	if ns.Annotations == nil {
+		ns.Annotations = map[string]string{}
+	}
+	ns.Annotations["namespace.cozystack.io/host"] = host
+
+	if err := c.Update(context.TODO(), &ns); err != nil {
+		return fmt.Errorf("failed to update namespace: %w", err)
+	}
+
+	return nil
+}

internal/controller/workload_controller.go

@@ -0,0 +1,99 @@
+package controller
+
+import (
+	"context"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
+)
+
+// WorkloadMonitorReconciler reconciles a WorkloadMonitor object
+type WorkloadReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+}
+
+func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+	w := &cozyv1alpha1.Workload{}
+	err := r.Get(ctx, req.NamespacedName, w)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return ctrl.Result{}, nil
+		}
+		logger.Error(err, "Unable to fetch Workload")
+		return ctrl.Result{}, err
+	}
+
+	// it's being deleted, nothing to handle
+	if w.DeletionTimestamp != nil {
+		return ctrl.Result{}, nil
+	}
+
+	t := getMonitoredObject(w)
+
+	if t == nil {
+		err = r.Delete(ctx, w)
+		if err != nil {
+			logger.Error(err, "failed to delete workload")
+		}
+		return ctrl.Result{}, err
+	}
+
+	err = r.Get(ctx, types.NamespacedName{Name: t.GetName(), Namespace: t.GetNamespace()}, t)
+
+	// found object, nothing to do
+	if err == nil {
+		return ctrl.Result{}, nil
+	}
+
+	// error getting object but not 404 -- requeue
+	if !apierrors.IsNotFound(err) {
+		logger.Error(err, "failed to get dependent object", "kind", t.GetObjectKind(), "dependent-object-name", t.GetName())
+		return ctrl.Result{}, err
+	}
+
+	err = r.Delete(ctx, w)
+	if err != nil {
+		logger.Error(err, "failed to delete workload")
+	}
+	return ctrl.Result{}, err
+}
+
+// SetupWithManager registers our controller with the Manager and sets up watches.
+func (r *WorkloadReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		// Watch WorkloadMonitor objects
+		For(&cozyv1alpha1.Workload{}).
+		Complete(r)
+}
+
+func getMonitoredObject(w *cozyv1alpha1.Workload) client.Object {
+	switch {
+	case strings.HasPrefix(w.Name, "pvc-"):
+		obj := &corev1.PersistentVolumeClaim{}
+		obj.Name = strings.TrimPrefix(w.Name, "pvc-")
+		obj.Namespace = w.Namespace
+		return obj
+	case strings.HasPrefix(w.Name, "svc-"):
+		obj := &corev1.Service{}
+		obj.Name = strings.TrimPrefix(w.Name, "svc-")
+		obj.Namespace = w.Namespace
+		return obj
+	case strings.HasPrefix(w.Name, "pod-"):
+		obj := &corev1.Pod{}
+		obj.Name = strings.TrimPrefix(w.Name, "pod-")
+		obj.Namespace = w.Namespace
+		return obj
+	}
+	var obj client.Object
+	return obj
+}

internal/controller/workload_controller_test.go

@@ -0,0 +1,26 @@
+package controller
+
+import (
+	"testing"
+
+	cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func TestUnprefixedMonitoredObjectReturnsNil(t *testing.T) {
+	w := &cozyv1alpha1.Workload{}
+	w.Name = "unprefixed-name"
+	obj := getMonitoredObject(w)
+	if obj != nil {
+		t.Errorf(`getMonitoredObject(&Workload{Name: "%s"}) == %v, want nil`, w.Name, obj)
+	}
+}
+
+func TestPodMonitoredObject(t *testing.T) {
+	w := &cozyv1alpha1.Workload{}
+	w.Name = "pod-mypod"
+	obj := getMonitoredObject(w)
+	if pod, ok := obj.(*corev1.Pod); !ok || pod.Name != "mypod" {
+		t.Errorf(`getMonitoredObject(&Workload{Name: "%s"}) == %v, want &Pod{Name: "mypod"}`, w.Name, obj)
+	}
+}

internal/controller/workloadmonitor_controller.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"sort"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -33,6 +34,17 @@ type WorkloadMonitorReconciler struct {
 // +kubebuilder:rbac:groups=cozystack.io,resources=workloads,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=cozystack.io,resources=workloads/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch
+
+// isServiceReady checks if the service has an external IP bound
+func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool {
+	return len(svc.Status.LoadBalancer.Ingress) > 0
+}
+
+// isPVCReady checks if the PVC is bound
+func (r *WorkloadMonitorReconciler) isPVCReady(pvc *corev1.PersistentVolumeClaim) bool {
+	return pvc.Status.Phase == corev1.ClaimBound
+}
 
 // isPodReady checks if the Pod is in the Ready condition.
 func (r *WorkloadMonitorReconciler) isPodReady(pod *corev1.Pod) bool {
@@ -88,6 +100,110 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) {
 	obj.SetOwnerReferences(owners)
 }
 
+// reconcileServiceForMonitor creates or updates a Workload object for the given Service and WorkloadMonitor.
+func (r *WorkloadMonitorReconciler) reconcileServiceForMonitor(
+	ctx context.Context,
+	monitor *cozyv1alpha1.WorkloadMonitor,
+	svc corev1.Service,
+) error {
+	logger := log.FromContext(ctx)
+	workload := &cozyv1alpha1.Workload{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("svc-%s", svc.Name),
+			Namespace: svc.Namespace,
+		},
+	}
+
+	resources := make(map[string]resource.Quantity)
+
+	quantity := resource.MustParse("0")
+
+	for _, ing := range svc.Status.LoadBalancer.Ingress {
+		if ing.IP != "" {
+			quantity.Add(resource.MustParse("1"))
+		}
+	}
+
+	var resourceLabel string
+	if svc.Annotations != nil {
+		var ok bool
+		resourceLabel, ok = svc.Annotations["metallb.universe.tf/ip-allocated-from-pool"]
+		if !ok {
+			resourceLabel = "default"
+		}
+	}
+	resourceLabel = fmt.Sprintf("%s.ipaddresspool.metallb.io/requests.ipaddresses", resourceLabel)
+	resources[resourceLabel] = quantity
+
+	_, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error {
+		// Update owner references with the new monitor
+		updateOwnerReferences(workload.GetObjectMeta(), monitor)
+
+		workload.Labels = svc.Labels
+
+		// Fill Workload status fields:
+		workload.Status.Kind = monitor.Spec.Kind
+		workload.Status.Type = monitor.Spec.Type
+		workload.Status.Resources = resources
+		workload.Status.Operational = r.isServiceReady(&svc)
+
+		return nil
+	})
+	if err != nil {
+		logger.Error(err, "Failed to CreateOrUpdate Workload", "workload", workload.Name)
+		return err
+	}
+
+	return nil
+}
+
+// reconcilePVCForMonitor creates or updates a Workload object for the given PVC and WorkloadMonitor.
+func (r *WorkloadMonitorReconciler) reconcilePVCForMonitor(
+	ctx context.Context,
+	monitor *cozyv1alpha1.WorkloadMonitor,
+	pvc corev1.PersistentVolumeClaim,
+) error {
+	logger := log.FromContext(ctx)
+	workload := &cozyv1alpha1.Workload{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("pvc-%s", pvc.Name),
+			Namespace: pvc.Namespace,
+		},
+	}
+
+	resources := make(map[string]resource.Quantity)
+
+	for resourceName, resourceQuantity := range pvc.Status.Capacity {
+		storageClass := "default"
+		if pvc.Spec.StorageClassName != nil || *pvc.Spec.StorageClassName == "" {
+			storageClass = *pvc.Spec.StorageClassName
+		}
+		resourceLabel := fmt.Sprintf("%s.storageclass.storage.k8s.io/requests.%s", storageClass, resourceName.String())
+		resources[resourceLabel] = resourceQuantity
+	}
+
+	_, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error {
+		// Update owner references with the new monitor
+		updateOwnerReferences(workload.GetObjectMeta(), monitor)
+
+		workload.Labels = pvc.Labels
+
+		// Fill Workload status fields:
+		workload.Status.Kind = monitor.Spec.Kind
+		workload.Status.Type = monitor.Spec.Type
+		workload.Status.Resources = resources
+		workload.Status.Operational = r.isPVCReady(&pvc)
+
+		return nil
+	})
+	if err != nil {
+		logger.Error(err, "Failed to CreateOrUpdate Workload", "workload", workload.Name)
+		return err
+	}
+
+	return nil
+}
+
 // reconcilePodForMonitor creates or updates a Workload object for the given Pod and WorkloadMonitor.
 func (r *WorkloadMonitorReconciler) reconcilePodForMonitor(
 	ctx context.Context,
@@ -96,15 +212,12 @@ func (r *WorkloadMonitorReconciler) reconcilePodForMonitor(
 ) error {
 	logger := log.FromContext(ctx)
 
-	// Combine both init containers and normal containers to sum resources properly
-	combinedContainers := append(pod.Spec.InitContainers, pod.Spec.Containers...)
-
-	// totalResources will store the sum of all container resource limits
+	// totalResources will store the sum of all container resource requests
 	totalResources := make(map[string]resource.Quantity)
 
-	// Iterate over all containers to aggregate their Limits
-	for _, container := range combinedContainers {
-		for name, qty := range container.Resources.Limits {
+	// Iterate over all containers to aggregate their requests
+	for _, container := range pod.Spec.Containers {
+		for name, qty := range container.Resources.Requests {
 			if existing, exists := totalResources[name.String()]; exists {
 				existing.Add(qty)
 				totalResources[name.String()] = existing
@@ -133,7 +246,7 @@ func (r *WorkloadMonitorReconciler) reconcilePodForMonitor(
 
 	workload := &cozyv1alpha1.Workload{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      pod.Name,
+			Name:      fmt.Sprintf("pod-%s", pod.Name),
 			Namespace: pod.Namespace,
 		},
 	}
@@ -205,6 +318,45 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		}
 	}
 
+	pvcList := &corev1.PersistentVolumeClaimList{}
+	if err := r.List(
+		ctx,
+		pvcList,
+		client.InNamespace(monitor.Namespace),
+		client.MatchingLabels(monitor.Spec.Selector),
+	); err != nil {
+		logger.Error(err, "Unable to list PVCs for WorkloadMonitor", "monitor", monitor.Name)
+		return ctrl.Result{}, err
+	}
+
+	for _, pvc := range pvcList.Items {
+		if err := r.reconcilePVCForMonitor(ctx, monitor, pvc); err != nil {
+			logger.Error(err, "Failed to reconcile Workload for PVC", "PVC", pvc.Name)
+			continue
+		}
+	}
+
+	svcList := &corev1.ServiceList{}
+	if err := r.List(
+		ctx,
+		svcList,
+		client.InNamespace(monitor.Namespace),
+		client.MatchingLabels(monitor.Spec.Selector),
+	); err != nil {
+		logger.Error(err, "Unable to list Services for WorkloadMonitor", "monitor", monitor.Name)
+		return ctrl.Result{}, err
+	}
+
+	for _, svc := range svcList.Items {
+		if svc.Spec.Type != corev1.ServiceTypeLoadBalancer {
+			continue
+		}
+		if err := r.reconcileServiceForMonitor(ctx, monitor, svc); err != nil {
+			logger.Error(err, "Failed to reconcile Workload for Service", "Service", svc.Name)
+			continue
+		}
+	}
+
 	// Update WorkloadMonitor status based on observed pods
 	monitor.Status.ObservedReplicas = observedReplicas
 	monitor.Status.AvailableReplicas = availableReplicas
@@ -233,41 +385,51 @@ func (r *WorkloadMonitorReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		// Also watch Pod objects and map them back to WorkloadMonitor if labels match
 		Watches(
 			&corev1.Pod{},
-			handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
-				pod, ok := obj.(*corev1.Pod)
-				if !ok {
-					return nil
-				}
-
-				var monitorList cozyv1alpha1.WorkloadMonitorList
-				// List all WorkloadMonitors in the same namespace
-				if err := r.List(ctx, &monitorList, client.InNamespace(pod.Namespace)); err != nil {
-					return nil
-				}
-
-				// Match each monitor's selector with the Pod's labels
-				var requests []reconcile.Request
-				for _, m := range monitorList.Items {
-					matches := true
-					for k, v := range m.Spec.Selector {
-						if podVal, exists := pod.Labels[k]; !exists || podVal != v {
-							matches = false
-							break
-						}
-					}
-					if matches {
-						requests = append(requests, reconcile.Request{
-							NamespacedName: types.NamespacedName{
-								Namespace: m.Namespace,
-								Name:      m.Name,
-							},
-						})
-					}
-				}
-				return requests
-			}),
+			handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(&corev1.Pod{}, r.Client)),
+		).
+		// Watch PVCs as well
+		Watches(
+			&corev1.PersistentVolumeClaim{},
+			handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(&corev1.PersistentVolumeClaim{}, r.Client)),
 		).
 		// Watch for changes to Workload objects we create (owned by WorkloadMonitor)
 		Owns(&cozyv1alpha1.Workload{}).
 		Complete(r)
 }
+
+func mapObjectToMonitor[T client.Object](_ T, c client.Client) func(ctx context.Context, obj client.Object) []reconcile.Request {
+	return func(ctx context.Context, obj client.Object) []reconcile.Request {
+		concrete, ok := obj.(T)
+		if !ok {
+			return nil
+		}
+
+		var monitorList cozyv1alpha1.WorkloadMonitorList
+		// List all WorkloadMonitors in the same namespace
+		if err := c.List(ctx, &monitorList, client.InNamespace(concrete.GetNamespace())); err != nil {
+			return nil
+		}
+
+		labels := concrete.GetLabels()
+		// Match each monitor's selector with the Pod's labels
+		var requests []reconcile.Request
+		for _, m := range monitorList.Items {
+			matches := true
+			for k, v := range m.Spec.Selector {
+				if labelVal, exists := labels[k]; !exists || labelVal != v {
+					matches = false
+					break
+				}
+			}
+			if matches {
+				requests = append(requests, reconcile.Request{
+					NamespacedName: types.NamespacedName{
+						Namespace: m.Namespace,
+						Name:      m.Name,
+					},
+				})
+			}
+		}
+		return requests
+	}
+}

manifests/cozystack-installer.yaml

@@ -1,105 +0,0 @@
----
-# Source: cozy-installer/templates/cozystack.yaml
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: cozy-system
-  labels:
-    cozystack.io/system: "true"
-    pod-security.kubernetes.io/enforce: privileged
----
-# Source: cozy-installer/templates/cozystack.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: cozystack
-  namespace: cozy-system
----
-# Source: cozy-installer/templates/cozystack.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: cozystack
-subjects:
-- kind: ServiceAccount
-  name: cozystack
-  namespace: cozy-system
-roleRef:
-  kind: ClusterRole
-  name: cluster-admin
-  apiGroup: rbac.authorization.k8s.io
----
-# Source: cozy-installer/templates/cozystack.yaml
-apiVersion: v1
-kind: Service
-metadata:
-  name: cozystack
-  namespace: cozy-system
-spec:
-  ports:
-  - name: http
-    port: 80
-    targetPort: 8123
-  selector:
-    app: cozystack
-  type: ClusterIP
----
-# Source: cozy-installer/templates/cozystack.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: cozystack
-  namespace: cozy-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: cozystack
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxSurge: 0
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        app: cozystack
-    spec:
-      hostNetwork: true
-      serviceAccountName: cozystack
-      containers:
-      - name: cozystack
-        image: "ghcr.io/cozystack/cozystack/installer:v0.28.0"
-        env:
-        - name: KUBERNETES_SERVICE_HOST
-          value: localhost
-        - name: KUBERNETES_SERVICE_PORT
-          value: "7445"
-        - name: K8S_AWAIT_ELECTION_ENABLED
-          value: "1"
-        - name: K8S_AWAIT_ELECTION_NAME
-          value: cozystack
-        - name: K8S_AWAIT_ELECTION_LOCK_NAME
-          value: cozystack
-        - name: K8S_AWAIT_ELECTION_LOCK_NAMESPACE
-          value: cozy-system
-        - name: K8S_AWAIT_ELECTION_IDENTITY
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-      - name: assets
-        image: "ghcr.io/cozystack/cozystack/installer:v0.28.0"
-        command:
-        - /usr/bin/cozystack-assets-server
-        - "-dir=/cozystack/assets"
-        - "-address=:8123"
-        ports:
-        - name: http
-          containerPort: 8123
-      tolerations:
-      - key: "node.kubernetes.io/not-ready"
-        operator: "Exists"
-        effect: "NoSchedule"
-      - key: "node.cilium.io/agent-not-ready"
-        operator: "Exists"
-        effect: "NoSchedule"

packages/apps/Makefile

@@ -1,14 +1,8 @@
-OUT=../../_out/repos/apps
-TMP=../../_out/repos/apps/historical
+OUT=../_out/repos/apps
+TMP := $(shell mktemp -d)
 
 repo:
-	rm -rf "$(OUT)"
-	mkdir -p "$(OUT)"
-	awk '$$3 != "HEAD" {print "mkdir -p $(TMP)/" $$1 "-" $$2}' versions_map | sh -ex
-	awk '$$3 != "HEAD" {print "git archive " $$3 " " $$1 " | tar -xf- --strip-components=1 -C $(TMP)/" $$1 "-" $$2 }' versions_map | sh -ex
-	helm package -d "$(OUT)" $$(find . $(TMP) -mindepth 2 -maxdepth 2 -name Chart.yaml | awk 'sub("/Chart.yaml", "")' | sort -V)
-	cd "$(OUT)" && helm repo index . --url http://cozystack.cozy-system.svc/repos/apps
-	rm -rf "$(TMP)"
+	cd .. && ../hack/package_chart.sh apps $(OUT) $(TMP) library
 
 fix-chartnames:
 	find . -maxdepth 2 -name Chart.yaml  | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done

packages/apps/bucket/README.md

@@ -0,0 +1,3 @@
+# S3 bucket
+
+## Parameters

packages/apps/bucket/templates/helmrelease.yaml

@@ -11,7 +11,7 @@ spec:
         kind: HelmRepository
         name: cozystack-system
         namespace: cozy-system
-      version: '*'
+      version: '>= 0.0.0-0'
   interval: 1m0s
   timeout: 5m0s
   values:

packages/apps/bucket/values.schema.json

@@ -0,0 +1,5 @@
+{
+    "title": "Chart Values",
+    "type": "object",
+    "properties": {}
+}

packages/apps/bucket/values.yaml

@@ -0,0 +1 @@
+{}

packages/apps/clickhouse/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.6.2
+version: 0.9.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/clickhouse/Makefile

@@ -1,4 +1,4 @@
-CLICKHOUSE_BACKUP_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
+CLICKHOUSE_BACKUP_TAG = $(shell awk '$$0 ~ /^version:/ {print $$2}' Chart.yaml)
 
 include ../../../scripts/common-envs.mk
 include ../../../scripts/package.mk
@@ -7,8 +7,10 @@ generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md
 
 image:
-	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/clickhouse-backup \
+	docker buildx build images/clickhouse-backup \
 		--provenance false \
+		--builder=$(BUILDER) \
+		--platform=$(PLATFORM) \
 		--tag $(REGISTRY)/clickhouse-backup:$(call settag,$(CLICKHOUSE_BACKUP_TAG)) \
 		--cache-from type=registry,ref=$(REGISTRY)/clickhouse-backup:latest \
 		--cache-to type=inline \

packages/apps/clickhouse/README.md

@@ -1,32 +1,35 @@
 # Managed Clickhouse Service
 
+ClickHouse is an open source high-performance and column-oriented SQL database management system (DBMS).
+It is used for online analytical processing (OLAP).
+Cozystack platform uses Altinity operator to provide ClickHouse.
+
 ### How to restore backup:
 
-find snapshot:
-```
-restic -r s3:s3.example.org/clickhouse-backups/table_name snapshots
-```
+1. Find a snapshot:
+    ```
+    restic -r s3:s3.example.org/clickhouse-backups/table_name snapshots
+    ```
 
-restore:
-```
-restic -r s3:s3.example.org/clickhouse-backups/table_name restore latest --target /tmp/
-```
+2.  Restore it:
+    ```
+    restic -r s3:s3.example.org/clickhouse-backups/table_name restore latest --target /tmp/
+    ```
 
-more details:
-- https://itnext.io/restic-effective-backup-from-stdin-4bc1e8f083c1
+For more details, read [Restic: Effective Backup from Stdin](https://blog.aenix.io/restic-effective-backup-from-stdin-4bc1e8f083c1).
 
 ## Parameters
 
 ### Common parameters
 
-| Name             | Description                         | Value  |
-| ---------------- | ----------------------------------- | ------ |
-| `size`           | Persistent Volume size              | `10Gi` |
-| `logStorageSize` | Persistent Volume for logs size     | `2Gi`  |
-| `shards`         | Number of Clickhouse replicas       | `1`    |
-| `replicas`       | Number of Clickhouse shards         | `2`    |
-| `storageClass`   | StorageClass used to store the data | `""`   |
-| `logTTL`         | for query_log and query_thread_log  | `15`   |
+| Name             | Description                                              | Value  |
+| ---------------- | -------------------------------------------------------- | ------ |
+| `size`           | Size of Persistent Volume for data                       | `10Gi` |
+| `logStorageSize` | Size of Persistent Volume for logs                       | `2Gi`  |
+| `shards`         | Number of Clickhouse shards                              | `1`    |
+| `replicas`       | Number of Clickhouse replicas                            | `2`    |
+| `storageClass`   | StorageClass used to store the data                      | `""`   |
+| `logTTL`         | TTL (expiration time) for query_log and query_thread_log | `15`   |
 
 ### Configuration parameters
 
@@ -36,13 +39,32 @@ more details:
 
 ### Backup parameters
 
-| Name                     | Description                                    | Value                                                  |
-| ------------------------ | ---------------------------------------------- | ------------------------------------------------------ |
-| `backup.enabled`         | Enable pereiodic backups                       | `false`                                                |
-| `backup.s3Region`        | The AWS S3 region where backups are stored     | `us-east-1`                                            |
-| `backup.s3Bucket`        | The S3 bucket used for storing backups         | `s3.example.org/clickhouse-backups`                    |
-| `backup.schedule`        | Cron schedule for automated backups            | `0 2 * * *`                                            |
-| `backup.cleanupStrategy` | The strategy for cleaning up old backups       | `--keep-last=3 --keep-daily=3 --keep-within-weekly=1m` |
-| `backup.s3AccessKey`     | The access key for S3, used for authentication | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu`                     |
-| `backup.s3SecretKey`     | The secret key for S3, used for authentication | `ju3eum4dekeich9ahM1te8waeGai0oog`                     |
-| `backup.resticPassword`  | The password for Restic backup encryption      | `ChaXoveekoh6eigh4siesheeda2quai0`                     |
+| Name                     | Description                                                                 | Value                                                  |
+| ------------------------ | --------------------------------------------------------------------------- | ------------------------------------------------------ |
+| `backup.enabled`         | Enable periodic backups                                                     | `false`                                                |
+| `backup.s3Region`        | AWS S3 region where backups are stored                                      | `us-east-1`                                            |
+| `backup.s3Bucket`        | S3 bucket used for storing backups                                          | `s3.example.org/clickhouse-backups`                    |
+| `backup.schedule`        | Cron schedule for automated backups                                         | `0 2 * * *`                                            |
+| `backup.cleanupStrategy` | Retention strategy for cleaning up old backups                              | `--keep-last=3 --keep-daily=3 --keep-within-weekly=1m` |
+| `backup.s3AccessKey`     | Access key for S3, used for authentication                                  | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu`                     |
+| `backup.s3SecretKey`     | Secret key for S3, used for authentication                                  | `ju3eum4dekeich9ahM1te8waeGai0oog`                     |
+| `backup.resticPassword`  | Password for Restic backup encryption                                       | `ChaXoveekoh6eigh4siesheeda2quai0`                     |
+| `resources`              | Explicit CPU/memory resource requests and limits for the Clickhouse service | `{}`                                                   |
+| `resourcesPreset`        | Use a common resources preset when `resources` is not set explicitly.       | `nano`                                                 |
+
+
+In production environments, it's recommended to set `resources` explicitly.
+Example of `resources`:
+
+```yaml
+resources:
+  limits:
+    cpu: 4000m
+    memory: 4Gi
+  requests:
+    cpu: 100m
+    memory: 512Mi
+```
+
+Allowed values for `resourcesPreset` are `none`, `nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`.
+This value is ignored if `resources` value is set. 

packages/apps/clickhouse/charts/cozy-lib

@@ -0,0 +1 @@
+../../../library/cozy-lib

packages/apps/clickhouse/images/clickhouse-backup.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/clickhouse-backup:0.6.2@sha256:67dd53efa86b704fc5cb876aca055fef294b31ab67899b683a4821ea12582ea7
+ghcr.io/cozystack/cozystack/clickhouse-backup:0.9.2@sha256:3faf7a4cebf390b9053763107482de175aa0fdb88c1e77424fd81100b1c3a205

packages/apps/clickhouse/templates/_resources.tpl

@@ -0,0 +1,49 @@
+{{/*
+Copyright Broadcom, Inc. All Rights Reserved.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Return a resource request/limit object based on a given preset.
+These presets are for basic testing and not meant to be used in production
+{{ include "resources.preset" (dict "type" "nano") -}}
+*/}}
+{{- define "resources.preset" -}}
+{{- $presets := dict 
+  "nano" (dict 
+      "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi")
+   )
+  "micro" (dict 
+      "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi")
+   )
+  "small" (dict 
+      "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi")
+   )
+  "medium" (dict 
+      "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi")
+   )
+  "large" (dict 
+      "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi")
+   )
+  "xlarge" (dict 
+      "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi")
+   )
+  "2xlarge" (dict 
+      "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi")
+   )
+ }}
+{{- if hasKey $presets .type -}}
+{{- index $presets .type | toYaml -}}
+{{- else -}}
+{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" .type (join "," (keys $presets)) | fail -}}
+{{- end -}}
+{{- end -}}

packages/apps/clickhouse/templates/clickhouse.yaml

@@ -1,3 +1,5 @@
+{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
+{{- $clusterDomain := (index $cozyConfig.data "cluster-domain") | default "cozy.local" }}
 {{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (printf "%s-credentials" .Release.Name) }}
 {{- $passwords := dict }}
 {{- $users := .Values.users }}
@@ -32,7 +34,7 @@ kind: "ClickHouseInstallation"
 metadata:
   name: "{{ .Release.Name }}"
 spec:
-  namespaceDomainPattern:  "%s.svc.cozy.local"
+  namespaceDomainPattern:  "%s.svc.{{ $clusterDomain }}"
   defaults:
     templates:
       dataVolumeClaimTemplate: data-volume-template
@@ -92,6 +94,9 @@ spec:
   templates:
     volumeClaimTemplates:
       - name: data-volume-template
+        metadata:
+          labels:
+            app.kubernetes.io/instance: {{ .Release.Name }}
         spec:
           accessModes:
             - ReadWriteOnce
@@ -99,6 +104,9 @@ spec:
             requests:
               storage: {{ .Values.size }}
       - name: log-volume-template
+        metadata:
+          labels:
+            app.kubernetes.io/instance: {{ .Release.Name }}
         spec:
           accessModes:
             - ReadWriteOnce
@@ -107,6 +115,9 @@ spec:
               storage: {{ .Values.logStorageSize }}
     podTemplates:
       - name: clickhouse-per-host
+        metadata:
+          labels:
+            app.kubernetes.io/instance: {{ .Release.Name }}
         spec:
           affinity:
             podAntiAffinity:
@@ -121,6 +132,11 @@ spec:
           containers:
             - name: clickhouse
               image: clickhouse/clickhouse-server:24.9.2.42
+              {{- if .Values.resources }}
+              resources: {{- include "cozy-lib.resources.sanitize" (list .Values.resources $) | nindent 16 }}
+              {{- else if ne .Values.resourcesPreset "none" }}
+              resources: {{- include "cozy-lib.resources.preset" (list .Values.resourcesPreset $) | nindent 16 }}
+              {{- end }}
               volumeMounts:
                 - name: data-volume-template
                   mountPath: /var/lib/clickhouse
@@ -128,6 +144,9 @@ spec:
                   mountPath: /var/log/clickhouse-server
     serviceTemplates:
       - name: svc-template
+        metadata:
+          labels:
+            app.kubernetes.io/instance: {{ .Release.Name }}
         generateName: chendpoint-{chi}
         spec:
           ports:

packages/apps/clickhouse/templates/workloadmonitor.yaml

@@ -9,5 +9,5 @@ spec:
   kind: clickhouse
   type: clickhouse
   selector:
-    clickhouse.altinity.com/chi: {{ $.Release.Name }}
+    app.kubernetes.io/instance: {{ $.Release.Name }}
   version: {{ $.Chart.Version }}

packages/apps/clickhouse/values.schema.json

@@ -4,22 +4,22 @@
     "properties": {
         "size": {
             "type": "string",
-            "description": "Persistent Volume size",
+            "description": "Size of Persistent Volume for data",
             "default": "10Gi"
         },
         "logStorageSize": {
             "type": "string",
-            "description": "Persistent Volume for logs size",
+            "description": "Size of Persistent Volume for logs",
             "default": "2Gi"
         },
         "shards": {
             "type": "number",
-            "description": "Number of Clickhouse replicas",
+            "description": "Number of Clickhouse shards",
             "default": 1
         },
         "replicas": {
             "type": "number",
-            "description": "Number of Clickhouse shards",
+            "description": "Number of Clickhouse replicas",
             "default": 2
         },
         "storageClass": {
@@ -29,7 +29,7 @@
         },
         "logTTL": {
             "type": "number",
-            "description": "for query_log and query_thread_log",
+            "description": "TTL (expiration time) for query_log and query_thread_log",
             "default": 15
         },
         "backup": {
@@ -37,17 +37,17 @@
             "properties": {
                 "enabled": {
                     "type": "boolean",
-                    "description": "Enable pereiodic backups",
+                    "description": "Enable periodic backups",
                     "default": false
                 },
                 "s3Region": {
                     "type": "string",
-                    "description": "The AWS S3 region where backups are stored",
+                    "description": "AWS S3 region where backups are stored",
                     "default": "us-east-1"
                 },
                 "s3Bucket": {
                     "type": "string",
-                    "description": "The S3 bucket used for storing backups",
+                    "description": "S3 bucket used for storing backups",
                     "default": "s3.example.org/clickhouse-backups"
                 },
                 "schedule": {
@@ -57,25 +57,35 @@
                 },
                 "cleanupStrategy": {
                     "type": "string",
-                    "description": "The strategy for cleaning up old backups",
+                    "description": "Retention strategy for cleaning up old backups",
                     "default": "--keep-last=3 --keep-daily=3 --keep-within-weekly=1m"
                 },
                 "s3AccessKey": {
                     "type": "string",
-                    "description": "The access key for S3, used for authentication",
+                    "description": "Access key for S3, used for authentication",
                     "default": "oobaiRus9pah8PhohL1ThaeTa4UVa7gu"
                 },
                 "s3SecretKey": {
                     "type": "string",
-                    "description": "The secret key for S3, used for authentication",
+                    "description": "Secret key for S3, used for authentication",
                     "default": "ju3eum4dekeich9ahM1te8waeGai0oog"
                 },
                 "resticPassword": {
                     "type": "string",
-                    "description": "The password for Restic backup encryption",
+                    "description": "Password for Restic backup encryption",
                     "default": "ChaXoveekoh6eigh4siesheeda2quai0"
                 }
             }
+        },
+        "resources": {
+            "type": "object",
+            "description": "Explicit CPU/memory resource requests and limits for the Clickhouse service",
+            "default": {}
+        },
+        "resourcesPreset": {
+            "type": "string",
+            "description": "Use a common resources preset when `resources` is not set explicitly.",
+            "default": "nano"
         }
     }
 }

packages/apps/clickhouse/values.yaml

@@ -1,11 +1,11 @@
 ## @section Common parameters
 
-## @param size Persistent Volume size
-## @param logStorageSize Persistent Volume for logs size
-## @param shards Number of Clickhouse replicas
-## @param replicas Number of Clickhouse shards
+## @param size Size of Persistent Volume for data
+## @param logStorageSize Size of Persistent Volume for logs
+## @param shards Number of Clickhouse shards
+## @param replicas Number of Clickhouse replicas
 ## @param storageClass StorageClass used to store the data
-## @param logTTL for query_log and query_thread_log
+## @param logTTL TTL (expiration time) for query_log and query_thread_log
 ##
 size: 10Gi
 logStorageSize: 2Gi
@@ -29,14 +29,14 @@ users: {}
 
 ## @section Backup parameters
 
-## @param backup.enabled Enable pereiodic backups
-## @param backup.s3Region The AWS S3 region where backups are stored
-## @param backup.s3Bucket The S3 bucket used for storing backups
+## @param backup.enabled Enable periodic backups
+## @param backup.s3Region AWS S3 region where backups are stored
+## @param backup.s3Bucket S3 bucket used for storing backups
 ## @param backup.schedule Cron schedule for automated backups
-## @param backup.cleanupStrategy The strategy for cleaning up old backups
-## @param backup.s3AccessKey The access key for S3, used for authentication
-## @param backup.s3SecretKey The secret key for S3, used for authentication
-## @param backup.resticPassword The password for Restic backup encryption
+## @param backup.cleanupStrategy Retention strategy for cleaning up old backups
+## @param backup.s3AccessKey Access key for S3, used for authentication
+## @param backup.s3SecretKey Secret key for S3, used for authentication
+## @param backup.resticPassword Password for Restic backup encryption
 backup:
   enabled: false
   s3Region: us-east-1
@@ -46,3 +46,16 @@ backup:
   s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
   s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
   resticPassword: ChaXoveekoh6eigh4siesheeda2quai0
+
+## @param resources Explicit CPU/memory resource requests and limits for the Clickhouse service
+resources: {}
+ # resources:
+ #   limits:
+ #     cpu: 4000m
+ #     memory: 4Gi
+ #   requests:
+ #     cpu: 100m
+ #     memory: 512Mi
+
+## @param resourcesPreset Use a common resources preset when `resources` is not set explicitly.
+resourcesPreset: "nano"

packages/apps/ferretdb/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.2
+version: 0.6.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/ferretdb/README.md

@@ -21,15 +21,17 @@
 
 ### Backup parameters
 
-| Name                     | Description                                    | Value                                                  |
-| ------------------------ | ---------------------------------------------- | ------------------------------------------------------ |
-| `backup.enabled`         | Enable pereiodic backups                       | `false`                                                |
-| `backup.s3Region`        | The AWS S3 region where backups are stored     | `us-east-1`                                            |
-| `backup.s3Bucket`        | The S3 bucket used for storing backups         | `s3.example.org/postgres-backups`                      |
-| `backup.schedule`        | Cron schedule for automated backups            | `0 2 * * *`                                            |
-| `backup.cleanupStrategy` | The strategy for cleaning up old backups       | `--keep-last=3 --keep-daily=3 --keep-within-weekly=1m` |
-| `backup.s3AccessKey`     | The access key for S3, used for authentication | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu`                     |
-| `backup.s3SecretKey`     | The secret key for S3, used for authentication | `ju3eum4dekeich9ahM1te8waeGai0oog`                     |
-| `backup.resticPassword`  | The password for Restic backup encryption      | `ChaXoveekoh6eigh4siesheeda2quai0`                     |
+| Name                     | Description                                                                                                                                                                                                       | Value                                                  |
+| ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ |
+| `backup.enabled`         | Enable pereiodic backups                                                                                                                                                                                          | `false`                                                |
+| `backup.s3Region`        | The AWS S3 region where backups are stored                                                                                                                                                                        | `us-east-1`                                            |
+| `backup.s3Bucket`        | The S3 bucket used for storing backups                                                                                                                                                                            | `s3.example.org/postgres-backups`                      |
+| `backup.schedule`        | Cron schedule for automated backups                                                                                                                                                                               | `0 2 * * *`                                            |
+| `backup.cleanupStrategy` | The strategy for cleaning up old backups                                                                                                                                                                          | `--keep-last=3 --keep-daily=3 --keep-within-weekly=1m` |
+| `backup.s3AccessKey`     | The access key for S3, used for authentication                                                                                                                                                                    | `oobaiRus9pah8PhohL1ThaeTa4UVa7gu`                     |
+| `backup.s3SecretKey`     | The secret key for S3, used for authentication                                                                                                                                                                    | `ju3eum4dekeich9ahM1te8waeGai0oog`                     |
+| `backup.resticPassword`  | The password for Restic backup encryption                                                                                                                                                                         | `ChaXoveekoh6eigh4siesheeda2quai0`                     |
+| `resources`              | Resources                                                                                                                                                                                                         | `{}`                                                   |
+| `resourcesPreset`        | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `nano`                                                 |
 
 

packages/apps/ferretdb/charts/cozy-lib

@@ -0,0 +1 @@
+../../../library/cozy-lib

packages/apps/ferretdb/images/postgres-backup.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/postgres-backup:0.9.0@sha256:2b6ba87f5688a439bd2ac12835a5ab9e601feb15c0c44ed0d9ca48cec7c52521
+ghcr.io/cozystack/cozystack/postgres-backup:0.12.1@sha256:10179ed56457460d95cd5708db2a00130901255fa30c4dd76c65d2ef5622b61f

packages/apps/ferretdb/templates/_resources.tpl

@@ -0,0 +1,49 @@
+{{/*
+Copyright Broadcom, Inc. All Rights Reserved.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Return a resource request/limit object based on a given preset.
+These presets are for basic testing and not meant to be used in production
+{{ include "resources.preset" (dict "type" "nano") -}}
+*/}}
+{{- define "resources.preset" -}}
+{{- $presets := dict 
+  "nano" (dict 
+      "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi")
+   )
+  "micro" (dict 
+      "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi")
+   )
+  "small" (dict 
+      "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi")
+   )
+  "medium" (dict 
+      "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi")
+   )
+  "large" (dict 
+      "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi")
+   )
+  "xlarge" (dict 
+      "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi")
+   )
+  "2xlarge" (dict 
+      "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi")
+   )
+ }}
+{{- if hasKey $presets .type -}}
+{{- index $presets .type | toYaml -}}
+{{- else -}}
+{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" .type (join "," (keys $presets)) | fail -}}
+{{- end -}}
+{{- end -}}

packages/apps/ferretdb/templates/external-svc.yaml

@@ -2,6 +2,8 @@ apiVersion: v1
 kind: Service
 metadata:
   name: {{ .Release.Name }}
+  labels:
+    app.kubernetes.io/instance: {{ .Release.Name }}
 spec:
   type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }}
   {{- if .Values.external }}

packages/apps/ferretdb/templates/ferretdb.yaml

@@ -12,6 +12,7 @@ spec:
     metadata:
       labels:
         app: {{ .Release.Name }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
     spec:
       containers:
       - name: ferretdb

packages/apps/ferretdb/templates/postgres.yaml

@@ -11,11 +11,18 @@ spec:
   {{- $rawConstraints := get $configMap.data "globalAppTopologySpreadConstraints" }}
   {{- if $rawConstraints }}
   {{- $rawConstraints | fromYaml | toYaml | nindent 2 }}
+    labelSelector:
+      matchLabels:
+        cnpg.io/cluster: {{ .Release.Name }}-postgres
   {{- end }}
   {{- end }}
   minSyncReplicas: {{ .Values.quorum.minSyncReplicas }}
   maxSyncReplicas: {{ .Values.quorum.maxSyncReplicas }}
-
+  {{- if .Values.resources }}
+  resources: {{- include "cozy-lib.resources.sanitize" (list .Values.resources $) | nindent 4 }}
+  {{- else if ne .Values.resourcesPreset "none" }}
+  resources: {{- include "cozy-lib.resources.preset" (list .Values.resourcesPreset $) | nindent 4 }}
+  {{- end }}
   monitoring:
     enablePodMonitor: true
 
@@ -28,6 +35,7 @@ spec:
   inheritedMetadata:
     labels:
       policy.cozystack.io/allow-to-apiserver: "true"
+      app.kubernetes.io/instance: {{ .Release.Name }}
 
   {{- if .Values.users }}
   managed:

packages/apps/ferretdb/templates/workloadmonitor.yaml

@@ -9,5 +9,5 @@ spec:
   kind: ferretdb
   type: ferretdb
   selector:
-    app: {{ $.Release.Name }}
+    app.kubernetes.io/instance: {{ $.Release.Name }}
   version: {{ $.Chart.Version }}

packages/apps/ferretdb/values.schema.json

@@ -81,6 +81,16 @@
                     "default": "ChaXoveekoh6eigh4siesheeda2quai0"
                 }
             }
+        },
+        "resources": {
+            "type": "object",
+            "description": "Resources",
+            "default": {}
+        },
+        "resourcesPreset": {
+            "type": "string",
+            "description": "Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).",
+            "default": "nano"
         }
     }
 }

packages/apps/ferretdb/values.yaml

@@ -48,3 +48,16 @@ backup:
   s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
   s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
   resticPassword: ChaXoveekoh6eigh4siesheeda2quai0
+
+## @param resources Resources
+resources: {}
+ # resources:
+ #   limits:
+ #     cpu: 4000m
+ #     memory: 4Gi
+ #   requests:
+ #     cpu: 100m
+ #     memory: 512Mi
+ 
+## @param resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).
+resourcesPreset: "nano"

packages/apps/http-cache/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.1
+version: 0.5.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/http-cache/Makefile

@@ -6,8 +6,10 @@ include ../../../scripts/package.mk
 image: image-nginx
 
 image-nginx:
-	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/nginx-cache \
+	docker buildx build images/nginx-cache \
 		--provenance false \
+		--builder=$(BUILDER) \
+		--platform=$(PLATFORM) \
 		--tag $(REGISTRY)/nginx-cache:$(call settag,$(NGINX_CACHE_TAG)) \
 		--cache-from type=registry,ref=$(REGISTRY)/nginx-cache:latest \
 		--cache-to type=inline \

packages/apps/http-cache/README.md

@@ -60,13 +60,17 @@ VTS module shows wrong upstream resonse time
 
 ### Common parameters
 
-| Name               | Description                                     | Value   |
-| ------------------ | ----------------------------------------------- | ------- |
-| `external`         | Enable external access from outside the cluster | `false` |
-| `size`             | Persistent Volume size                          | `10Gi`  |
-| `storageClass`     | StorageClass used to store the data             | `""`    |
-| `haproxy.replicas` | Number of HAProxy replicas                      | `2`     |
-| `nginx.replicas`   | Number of Nginx replicas                        | `2`     |
+| Name                      | Description                                                                                                                                                                                                       | Value   |
+| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
+| `external`                | Enable external access from outside the cluster                                                                                                                                                                   | `false` |
+| `size`                    | Persistent Volume size                                                                                                                                                                                            | `10Gi`  |
+| `storageClass`            | StorageClass used to store the data                                                                                                                                                                               | `""`    |
+| `haproxy.replicas`        | Number of HAProxy replicas                                                                                                                                                                                        | `2`     |
+| `nginx.replicas`          | Number of Nginx replicas                                                                                                                                                                                          | `2`     |
+| `haproxy.resources`       | Resources                                                                                                                                                                                                         | `{}`    |
+| `haproxy.resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `nano`  |
+| `nginx.resources`         | Resources                                                                                                                                                                                                         | `{}`    |
+| `nginx.resourcesPreset`   | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `nano`  |
 
 ### Configuration parameters
 

packages/apps/http-cache/charts/cozy-lib

@@ -0,0 +1 @@
+../../../library/cozy-lib

packages/apps/http-cache/images/nginx-cache.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/nginx-cache:0.3.1@sha256:2b82eae28239ca0f9968602c69bbb752cd2a5818e64934ccd06cb91d95d019c7
+ghcr.io/cozystack/cozystack/nginx-cache:0.5.1@sha256:50ac1581e3100bd6c477a71161cb455a341ffaf9e5e2f6086802e4e25271e8af

packages/apps/http-cache/images/nginx-cache/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04 as stage
+FROM ubuntu:22.04 AS stage
 
 ARG NGINX_VERSION=1.25.3
 ARG IP2LOCATION_C_VERSION=8.6.1
@@ -9,11 +9,15 @@ ARG FIFTYONEDEGREES_NGINX_VERSION=3.2.21.1
 ARG NGINX_CACHE_PURGE_VERSION=2.5.3
 ARG NGINX_VTS_VERSION=0.2.2
 
+ARG TARGETOS
+ARG TARGETARCH
+
 # Install required packages for development
-RUN apt-get update -q \
- && apt-get install -yq \
+RUN apt update -q \
+ && apt install -yq --no-install-recommends \
+    ca-certificates \
     unzip \
-    autoconf \
+    automake \
     build-essential \
     libtool \
     libpcre3 \
@@ -68,7 +72,7 @@ RUN checkinstall \
   --default \
   --pkgname=ip2location-c \
   --pkgversion=${IP2LOCATION_C_VERSION} \
-  --pkgarch=amd64 \
+  --pkgarch=${TARGETARCH} \
   --pkggroup=lib \
   --pkgsource="https://github.com/chrislim2888/IP2Location-C-Library" \
   --maintainer="Eduard Generalov <eduard@generalov.net>" \
@@ -97,7 +101,7 @@ RUN checkinstall \
   --default \
   --pkgname=ip2proxy-c \
   --pkgversion=${IP2PROXY_C_VERSION} \
-  --pkgarch=amd64 \
+  --pkgarch=${TARGETARCH} \
   --pkggroup=lib \
   --pkgsource="https://github.com/ip2location/ip2proxy-c" \
   --maintainer="Eduard Generalov <eduard@generalov.net>" \
@@ -144,7 +148,7 @@ RUN checkinstall \
   --default \
   --pkgname=nginx \
   --pkgversion=$VERS \
-  --pkgarch=amd64 \
+  --pkgarch=${TARGETARCH} \
   --pkggroup=web \
   --provides=nginx \
   --requires=ip2location-c,ip2proxy-c,libssl3,libc-bin,libc6,libzstd1,libpcre++0v5,libpcre16-3,libpcre2-8-0,libpcre3,libpcre32-3,libpcrecpp0v5,libmaxminddb0 \
@@ -165,10 +169,9 @@ COPY nginx-reloader.sh /usr/bin/nginx-reloader.sh
 RUN set -x \
  && groupadd --system --gid 101 nginx \
  && useradd --system --gid nginx --no-create-home --home /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \
- && apt update \
- && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates inotify-tools \
- && apt -y install /packages/*.deb \
- && apt-get clean \
+ && apt update -q \
+ && apt install -yq --no-install-recommends --no-install-suggests gnupg1 ca-certificates inotify-tools \
+ && apt install -y /packages/*.deb \
  && rm -rf /var/lib/apt/lists/* \
  && mkdir -p /var/lib/nginx /var/log/nginx \
  && ln -sf /dev/stdout /var/log/nginx/access.log \

packages/apps/http-cache/templates/_resources.tpl

@@ -0,0 +1,49 @@
+{{/*
+Copyright Broadcom, Inc. All Rights Reserved.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Return a resource request/limit object based on a given preset.
+These presets are for basic testing and not meant to be used in production
+{{ include "resources.preset" (dict "type" "nano") -}}
+*/}}
+{{- define "resources.preset" -}}
+{{- $presets := dict 
+  "nano" (dict 
+      "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi")
+   )
+  "micro" (dict 
+      "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi")
+   )
+  "small" (dict 
+      "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi")
+   )
+  "medium" (dict 
+      "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi")
+   )
+  "large" (dict 
+      "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi")
+   )
+  "xlarge" (dict 
+      "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi")
+   )
+  "2xlarge" (dict 
+      "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi")
+   )
+ }}
+{{- if hasKey $presets .type -}}
+{{- index $presets .type | toYaml -}}
+{{- else -}}
+{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" .type (join "," (keys $presets)) | fail -}}
+{{- end -}}
+{{- end -}}

packages/apps/http-cache/templates/haproxy/deployment.yaml

@@ -33,6 +33,11 @@ spec:
       containers:
       - image: haproxy:latest
         name: haproxy
+        {{- if .Values.haproxy.resources }}
+        resources: {{- include "cozy-lib.resources.sanitize" (list .Values.haproxy.resources $) | nindent 10 }}
+        {{- else if ne .Values.haproxy.resourcesPreset "none" }}
+        resources: {{- include "cozy-lib.resources.preset" (list .Values.haproxy.resourcesPreset $) | nindent 10 }}
+        {{- end }}
         ports:
         - containerPort: 8080
           name: http

packages/apps/http-cache/templates/nginx/deployment.yaml

@@ -52,6 +52,11 @@ spec:
       shareProcessNamespace: true
       containers:
       - name: nginx
+        {{- if $.Values.nginx.resources }}
+        resources: {{- include "cozy-lib.resources.sanitize" (list $.Values.nginx.resources $) | nindent 10 }}
+        {{- else if ne $.Values.nginx.resourcesPreset "none" }}
+        resources: {{- include "cozy-lib.resources.preset" (list $.Values.nginx.resourcesPreset $) | nindent 10 }}
+        {{- end }}
         image: "{{ $.Files.Get "images/nginx-cache.tag" | trim }}"
         readinessProbe:
           httpGet:
@@ -83,6 +88,13 @@ spec:
       - name: reloader
         image: "{{ $.Files.Get "images/nginx-cache.tag" | trim }}"
         command: ["/usr/bin/nginx-reloader.sh"]
+        resources:
+          limits:
+            cpu: 50m
+            memory: 50Mi
+          requests:
+            cpu: 50m
+            memory: 50Mi
         #command: ["sleep", "infinity"]
         volumeMounts:
         - mountPath: /etc/nginx/nginx.conf

packages/apps/http-cache/templates/workloadmonitor.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: cozystack.io/v1alpha1
+kind: WorkloadMonitor
+metadata:
+  name: {{ $.Release.Name }}-haproxy
+spec:
+  replicas: {{ .Values.haproxy.replicas }}
+  minReplicas: 1
+  kind: http-cache
+  type: http-cache
+  selector:
+    app: {{ $.Release.Name }}-haproxy
+  version: {{ $.Chart.Version }}
+---
+apiVersion: cozystack.io/v1alpha1
+kind: WorkloadMonitor
+metadata:
+  name: {{ $.Release.Name }}-nginx
+spec:
+  replicas: {{ .Values.nginx.replicas }}
+  minReplicas: 1
+  kind: http-cache
+  type: http-cache
+  selector:
+    app: {{ $.Release.Name }}-nginx-cache
+  version: {{ $.Chart.Version }}
+---
+apiVersion: cozystack.io/v1alpha1
+kind: WorkloadMonitor
+metadata:
+  name: {{ $.Release.Name }}
+spec:
+  replicas: {{ .Values.replicas }}
+  minReplicas: 1
+  kind: http-cache
+  type: http-cache
+  selector:
+    app.kubernetes.io/instance: {{ $.Release.Name }}
+  version: {{ $.Chart.Version }}

packages/apps/http-cache/values.schema.json

@@ -24,6 +24,16 @@
                     "type": "number",
                     "description": "Number of HAProxy replicas",
                     "default": 2
+                },
+                "resources": {
+                    "type": "object",
+                    "description": "Resources",
+                    "default": {}
+                },
+                "resourcesPreset": {
+                    "type": "string",
+                    "description": "Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).",
+                    "default": "nano"
                 }
             }
         },
@@ -34,6 +44,16 @@
                     "type": "number",
                     "description": "Number of Nginx replicas",
                     "default": 2
+                },
+                "resources": {
+                    "type": "object",
+                    "description": "Resources",
+                    "default": {}
+                },
+                "resourcesPreset": {
+                    "type": "string",
+                    "description": "Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).",
+                    "default": "nano"
                 }
             }
         },

packages/apps/http-cache/values.yaml

@@ -12,8 +12,32 @@ size: 10Gi
 storageClass: ""
 haproxy:
   replicas: 2
+  ## @param haproxy.resources Resources
+  resources: {}
+  # resources:
+  #   limits:
+  #     cpu: 4000m
+  #     memory: 4Gi
+  #   requests:
+  #     cpu: 100m
+  #     memory: 512Mi
+  
+  ## @param haproxy.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).
+  resourcesPreset: "nano"
 nginx:
   replicas: 2
+  ## @param nginx.resources Resources
+  resources: {}
+  # resources:
+  #   limits:
+  #     cpu: 4000m
+  #     memory: 4Gi
+  #   requests:
+  #     cpu: 100m
+  #     memory: 512Mi
+  
+  ## @param nginx.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).
+  resourcesPreset: "nano"
 
 ## @section Configuration parameters
 

packages/apps/kafka/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.3
+version: 0.6.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

packages/apps/kafka/README.md

@@ -4,15 +4,19 @@
 
 ### Common parameters
 
-| Name                     | Description                                     | Value   |
-| ------------------------ | ----------------------------------------------- | ------- |
-| `external`               | Enable external access from outside the cluster | `false` |
-| `kafka.size`             | Persistent Volume size for Kafka                | `10Gi`  |
-| `kafka.replicas`         | Number of Kafka replicas                        | `3`     |
-| `kafka.storageClass`     | StorageClass used to store the Kafka data       | `""`    |
-| `zookeeper.size`         | Persistent Volume size for ZooKeeper            | `5Gi`   |
-| `zookeeper.replicas`     | Number of ZooKeeper replicas                    | `3`     |
-| `zookeeper.storageClass` | StorageClass used to store the ZooKeeper data   | `""`    |
+| Name                        | Description                                                                                                                                                                                                       | Value   |
+| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
+| `external`                  | Enable external access from outside the cluster                                                                                                                                                                   | `false` |
+| `kafka.size`                | Persistent Volume size for Kafka                                                                                                                                                                                  | `10Gi`  |
+| `kafka.replicas`            | Number of Kafka replicas                                                                                                                                                                                          | `3`     |
+| `kafka.storageClass`        | StorageClass used to store the Kafka data                                                                                                                                                                         | `""`    |
+| `zookeeper.size`            | Persistent Volume size for ZooKeeper                                                                                                                                                                              | `5Gi`   |
+| `zookeeper.replicas`        | Number of ZooKeeper replicas                                                                                                                                                                                      | `3`     |
+| `zookeeper.storageClass`    | StorageClass used to store the ZooKeeper data                                                                                                                                                                     | `""`    |
+| `kafka.resources`           | Resources                                                                                                                                                                                                         | `{}`    |
+| `kafka.resourcesPreset`     | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `small` |
+| `zookeeper.resources`       | Resources                                                                                                                                                                                                         | `{}`    |
+| `zookeeper.resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `micro` |
 
 ### Configuration parameters
 

packages/apps/kafka/charts/cozy-lib

@@ -0,0 +1 @@
+../../../library/cozy-lib

packages/apps/kafka/templates/_resources.tpl

@@ -0,0 +1,49 @@
+{{/*
+Copyright Broadcom, Inc. All Rights Reserved.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Return a resource request/limit object based on a given preset.
+These presets are for basic testing and not meant to be used in production
+{{ include "resources.preset" (dict "type" "nano") -}}
+*/}}
+{{- define "resources.preset" -}}
+{{- $presets := dict 
+  "nano" (dict 
+      "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi")
+   )
+  "micro" (dict 
+      "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi")
+   )
+  "small" (dict 
+      "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi")
+   )
+  "medium" (dict 
+      "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi")
+   )
+  "large" (dict 
+      "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi")
+   )
+  "xlarge" (dict 
+      "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi")
+   )
+  "2xlarge" (dict 
+      "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi")
+   )
+ }}
+{{- if hasKey $presets .type -}}
+{{- index $presets .type | toYaml -}}
+{{- else -}}
+{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" .type (join "," (keys $presets)) | fail -}}
+{{- end -}}
+{{- end -}}

packages/apps/kafka/templates/kafka.yaml

@@ -8,6 +8,11 @@ metadata:
 spec:
   kafka:
     replicas: {{ .Values.kafka.replicas }}
+    {{- if .Values.kafka.resources }}
+    resources: {{- include "cozy-lib.resources.sanitize" (list .Values.kafka.resources $) | nindent 6 }}
+    {{- else if ne .Values.kafka.resourcesPreset "none" }}
+    resources: {{- include "cozy-lib.resources.preset" (list .Values.kafka.resourcesPreset $) | nindent 6 }}
+    {{- end }}
     listeners:
       - name: plain
         port: 9092
@@ -65,6 +70,11 @@ spec:
           key: kafka-metrics-config.yml
   zookeeper:
     replicas: {{ .Values.zookeeper.replicas }}
+    {{- if .Values.zookeeper.resources }}
+    resources: {{- include "cozy-lib.resources.sanitize" (list .Values.zookeeper.resources $) | nindent 6 }}
+    {{- else if ne .Values.zookeeper.resourcesPreset "none" }}
+    resources: {{- include "cozy-lib.resources.preset" (list .Values.zookeeper.resourcesPreset $) | nindent 6 }}
+    {{- end }}
     storage:
       type: persistent-claim
       {{- with .Values.zookeeper.size }}

packages/apps/kafka/values.schema.json

@@ -24,6 +24,16 @@
                     "type": "string",
                     "description": "StorageClass used to store the Kafka data",
                     "default": ""
+                },
+                "resources": {
+                    "type": "object",
+                    "description": "Resources",
+                    "default": {}
+                },
+                "resourcesPreset": {
+                    "type": "string",
+                    "description": "Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).",
+                    "default": "small"
                 }
             }
         },
@@ -44,6 +54,16 @@
                     "type": "string",
                     "description": "StorageClass used to store the ZooKeeper data",
                     "default": ""
+                },
+                "resources": {
+                    "type": "object",
+                    "description": "Resources",
+                    "default": {}
+                },
+                "resourcesPreset": {
+                    "type": "string",
+                    "description": "Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).",
+                    "default": "micro"
                 }
             }
         },

packages/apps/kafka/values.yaml

@@ -14,10 +14,35 @@ kafka:
   size: 10Gi
   replicas: 3
   storageClass: ""
+  ## @param kafka.resources Resources
+  resources: {}
+  # resources:
+  #   limits:
+  #     cpu: 4000m
+  #     memory: 4Gi
+  #   requests:
+  #     cpu: 100m
+  #     memory: 512Mi
+  
+  ## @param kafka.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).
+  resourcesPreset: "small"
+
 zookeeper:
   size: 5Gi
   replicas: 3
   storageClass: ""
+  ## @param zookeeper.resources Resources
+  resources: {}
+  # resources:
+  #   limits:
+  #     cpu: 4000m
+  #     memory: 4Gi
+  #   requests:
+  #     cpu: 100m
+  #     memory: 512Mi
+  
+  ## @param zookeeper.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production).
+  resourcesPreset: "micro"
 
 ## @section Configuration parameters
 

packages/apps/kubernetes/Chart.yaml

@@ -16,10 +16,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.15.2
+version: 0.23.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.30.1"
+appVersion: 1.32.4

packages/apps/kubernetes/Makefile

@@ -1,4 +1,4 @@
-UBUNTU_CONTAINER_DISK_TAG = v1.30.1
+KUBERNETES_VERSION = v1.32
 KUBERNETES_PKG_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
 
 include ../../../scripts/common-envs.mk
@@ -6,27 +6,36 @@ include ../../../scripts/package.mk
 
 generate:
 	readme-generator -v values.yaml -s values.schema.json -r README.md
+	yq -o json -i '.properties.controlPlane.properties.apiServer.properties.resourcesPreset.enum = ["none","nano","micro","small","medium","large","xlarge","2xlarge"]' values.schema.json
+	yq -o json -i '.properties.controlPlane.properties.controllerManager.properties.resourcesPreset.enum = ["none","nano","micro","small","medium","large","xlarge","2xlarge"]' values.schema.json
+	yq -o json -i '.properties.controlPlane.properties.scheduler.properties.resourcesPreset.enum = ["none","nano","micro","small","medium","large","xlarge","2xlarge"]' values.schema.json
+	yq -o json -i '.properties.controlPlane.properties.konnectivity.properties.server.properties.resourcesPreset.enum = ["none","nano","micro","small","medium","large","xlarge","2xlarge"]' values.schema.json
 
 image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler
 
 image-ubuntu-container-disk:
-	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/ubuntu-container-disk \
+	docker buildx build images/ubuntu-container-disk \
 		--provenance false \
-		--tag $(REGISTRY)/ubuntu-container-disk:$(call settag,$(UBUNTU_CONTAINER_DISK_TAG)) \
-		--tag $(REGISTRY)/ubuntu-container-disk:$(call settag,$(UBUNTU_CONTAINER_DISK_TAG)-$(TAG)) \
+		--builder=$(BUILDER) \
+		--platform=$(PLATFORM) \
+		--build-arg KUBERNETES_VERSION=${KUBERNETES_VERSION} \
+		--tag $(REGISTRY)/ubuntu-container-disk:$(call settag,$(KUBERNETES_VERSION)) \
+		--tag $(REGISTRY)/ubuntu-container-disk:$(call settag,$(KUBERNETES_VERSION)-$(TAG)) \
 		--cache-from type=registry,ref=$(REGISTRY)/ubuntu-container-disk:latest \
 		--cache-to type=inline \
 		--metadata-file images/ubuntu-container-disk.json \
 		--push=$(PUSH) \
 		--label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \
 		--load=$(LOAD)
-	echo "$(REGISTRY)/ubuntu-container-disk:$(call settag,$(UBUNTU_CONTAINER_DISK_TAG))@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk.json -o json -r)" \
+	echo "$(REGISTRY)/ubuntu-container-disk:$(call settag,$(KUBERNETES_VERSION))@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk.json -o json -r)" \
 		> images/ubuntu-container-disk.tag
 	rm -f images/ubuntu-container-disk.json
 
 image-kubevirt-cloud-provider:
-	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/kubevirt-cloud-provider \
+	docker buildx build images/kubevirt-cloud-provider \
 		--provenance false \
+		--builder=$(BUILDER) \
+		--platform=$(PLATFORM) \
 		--tag $(REGISTRY)/kubevirt-cloud-provider:$(call settag,$(KUBERNETES_PKG_TAG)) \
 		--tag $(REGISTRY)/kubevirt-cloud-provider:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \
 		--cache-from type=registry,ref=$(REGISTRY)/kubevirt-cloud-provider:latest \
@@ -40,8 +49,10 @@ image-kubevirt-cloud-provider:
 	rm -f images/kubevirt-cloud-provider.json
 
 image-kubevirt-csi-driver:
-	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/kubevirt-csi-driver \
+	docker buildx build images/kubevirt-csi-driver \
 		--provenance false \
+		--builder=$(BUILDER) \
+		--platform=$(PLATFORM) \
 		--tag $(REGISTRY)/kubevirt-csi-driver:$(call settag,$(KUBERNETES_PKG_TAG)) \
 		--tag $(REGISTRY)/kubevirt-csi-driver:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \
 		--cache-from type=registry,ref=$(REGISTRY)/kubevirt-csi-driver:latest \
@@ -56,8 +67,10 @@ image-kubevirt-csi-driver:
 
 
 image-cluster-autoscaler:
-	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/cluster-autoscaler \
+	docker buildx build images/cluster-autoscaler \
 		--provenance false \
+		--builder=$(BUILDER) \
+		--platform=$(PLATFORM) \
 		--tag $(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG)) \
 		--tag $(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \
 		--cache-from type=registry,ref=$(REGISTRY)/cluster-autoscaler:latest \

packages/apps/kubernetes/README.md

@@ -1,49 +1,200 @@
 # Managed Kubernetes Service
 
-## Overview
+## Managed Kubernetes in Cozystack
 
-The Managed Kubernetes Service offers a streamlined solution for efficiently managing server workloads. Kubernetes has emerged as the industry standard, providing a unified and accessible API, primarily utilizing YAML for configuration. This means that teams can easily understand and work with Kubernetes, streamlining infrastructure management.
+Whenever you want to deploy a custom containerized application in Cozystack, it's best to deploy it to a managed Kubernetes cluster.
 
-The Kubernetes leverages robust software design patterns, enabling continuous recovery in any scenario through the reconciliation method. Additionally, it ensures seamless scaling across a multitude of servers, addressing the challenges posed by complex and outdated APIs found in traditional virtualization platforms. This managed service eliminates the need for developing custom solutions or modifying source code, saving valuable time and effort.
+Cozystack deploys and manages Kubernetes-as-a-service as standalone applications within each tenant’s isolated environment.
+In Cozystack, such clusters are named tenant Kubernetes clusters, while the base Cozystack cluster is called a management or root cluster.
+Tenant clusters are fully separated from the management cluster and are intended for deploying tenant-specific or customer-developed applications.
 
-## Deployment Details
+Within a tenant cluster, users can take advantage of LoadBalancer services and easily provision physical volumes as needed.                               
+The control-plane operates within containers, while the worker nodes are deployed as virtual machines, all seamlessly managed by the application.
 
-The managed Kubernetes service deploys a standard Kubernetes cluster utilizing the Cluster API, Kamaji as control-plane provicer and the KubeVirt infrastructure provider. This ensures a consistent and reliable setup for workloads.
+## Why Use a Managed Kubernetes Cluster?
 
-Within this cluster, users can take advantage of LoadBalancer services and easily provision physical volumes as needed. The control-plane operates within containers, while the worker nodes are deployed as virtual machines, all seamlessly managed by the application.
+Kubernetes has emerged as the industry standard, providing a unified and accessible API, primarily utilizing YAML for configuration.
+This means that teams can easily understand and work with Kubernetes, streamlining infrastructure management.
 
-- Docs: https://github.com/clastix/kamaji
-- Docs: https://cluster-api.sigs.k8s.io/
-- GitHub: https://github.com/clastix/kamaji
-- GitHub: https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt
-- GitHub: https://github.com/kubevirt/csi-driver
+Kubernetes leverages robust software design patterns, enabling continuous recovery in any scenario through the reconciliation method.
+Additionally, it ensures seamless scaling across a multitude of servers,
+addressing the challenges posed by complex and outdated APIs found in traditional virtualization platforms.
+This managed service eliminates the need for developing custom solutions or modifying source code, saving valuable time and effort.
 
+The Managed Kubernetes Service in Cozystack offers a streamlined solution for efficiently managing server workloads.
 
-## How-Tos
+## Starting Work
 
-How to access to deployed cluster:
+Once the tenant Kubernetes cluster is ready, you can get a kubeconfig file to work with it.
+It can be done via UI or a `kubectl` request:
 
-```
-kubectl get secret -n <namespace> kubernetes-<clusterName>-admin-kubeconfig -o go-template='{{ printf "%s\n" (index .data "super-admin.conf" | base64decode) }}' > test
+-   Open the Cozystack dashboard, switch to your tenant, find and open the application page. Copy one of the config files from the **Secrets** section.
+-   Run the following command (using the management cluster kubeconfig):
+
+    ```bash
+    kubectl get secret -n tenant-<name> kubernetes-<clusterName>-admin-kubeconfig -o go-template='{{ printf "%s\n" (index .data "admin.conf" | base64decode) }}' > admin.conf
+    ```
+
+There are several kubeconfig options available:
+
+-   `admin.conf` — The standard kubeconfig for accessing your new cluster.
+    You can create additional Kubernetes users using this configuration.
+-   `admin.svc` — Same token as `admin.conf`, but with the API server address set to the internal service name.
+    Use it for applications running inside the cluster that need API access.
+-   `super-admin.conf` — Similar to `admin.conf`, but with extended administrative permissions.
+    Intended for troubleshooting and cluster maintenance tasks.
+-   `super-admin.svc` — Same as `super-admin.conf`, but pointing to the internal API server address.
+
+## Implementation Details
+
+A tenant Kubernetes cluster in Cozystack is essentially Kubernetes-in-Kubernetes.
+Deploying it involves the following components:
+
+-   **Kamaji Control Plane**: [Kamaji](https://kamaji.clastix.io/) is an open-source project that facilitates the deployment
+    of Kubernetes control planes as pods within a root cluster.
+    Each control plane pod includes essential components like `kube-apiserver`, `controller-manager`, and `scheduler`,
+    allowing for efficient multi-tenancy and resource utilization.
+
+-   **Etcd Cluster**: A dedicated etcd cluster is deployed using Ænix's [etcd-operator](https://github.com/aenix-io/etcd-operator).
+    It provides reliable and scalable key-value storage for the Kubernetes control plane.
+
+-   **Worker Nodes**: Virtual Machines are provisioned to serve as worker nodes using KubeVirt.
+    These nodes are configured to join the tenant Kubernetes cluster, enabling the deployment and management of workloads.
+
+-   **Cluster API**: Cozystack is using the [Kubernetes Cluster API](https://cluster-api.sigs.k8s.io/) to provision the components of a cluster.
+
+This architecture ensures isolated, scalable, and efficient tenant Kubernetes environments.
+
+See the reference for components utilized in this service:
+
+- [Kamaji Control Plane](https://kamaji.clastix.io)
+- [Kamaji — Cluster API](https://kamaji.clastix.io/cluster-api/)
+- [github.com/clastix/kamaji](https://github.com/clastix/kamaji)
+- [KubeVirt](https://kubevirt.io/)
+- [github.com/kubevirt/kubevirt](https://github.com/kubevirt/kubevirt)
+- [github.com/aenix-io/etcd-operator](https://github.com/aenix-io/etcd-operator)
+- [Kubernetes Cluster API](https://cluster-api.sigs.k8s.io/)
+- [github.com/kubernetes-sigs/cluster-api-provider-kubevirt](https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt)
+- [github.com/kubevirt/csi-driver](https://github.com/kubevirt/csi-driver)
+
+## Parameters
+
+### Common Parameters
+
+| Name                                | Description                                                                                                       | Value        |
+| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------- | ------------ |
+| `host`                              | Hostname used to access the Kubernetes cluster externally. Defaults to `<cluster-name>.<tenant-host>` when empty. | `""`         |
+| `controlPlane.replicas`             | Number of replicas for Kubernetes control-plane components.                                                       | `2`          |
+| `storageClass`                      | StorageClass used to store user data.                                                                             | `replicated` |
+| `useCustomSecretForPatchContainerd` | if true, for patch containerd will be used secret: {{ .Release.Name }}-patch-containerd                           | `false`      |
+| `nodeGroups`                        | nodeGroups configuration                                                                                          | `{}`         |
+
+### Cluster Addons
+
+| Name                                          | Description                                                                                                                                                                       | Value   |
+| --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
+| `addons.certManager.enabled`                  | Enable cert-manager, which automatically creates and manages SSL/TLS certificates.                                                                                                | `false` |
+| `addons.certManager.valuesOverride`           | Custom values to override                                                                                                                                                         | `{}`    |
+| `addons.cilium.valuesOverride`                | Custom values to override                                                                                                                                                         | `{}`    |
+| `addons.gatewayAPI.enabled`                   | Enable the Gateway API                                                                                                                                                            | `false` |
+| `addons.ingressNginx.enabled`                 | Enable the Ingress-NGINX controller (requires nodes labeled with the 'ingress-nginx' role).                                                                                       | `false` |
+| `addons.ingressNginx.valuesOverride`          | Custom values to override                                                                                                                                                         | `{}`    |
+| `addons.ingressNginx.hosts`                   | List of domain names that the parent cluster should route to this tenant cluster.                                                                                                 | `[]`    |
+| `addons.gpuOperator.enabled`                  | Enable the GPU-operator                                                                                                                                                           | `false` |
+| `addons.gpuOperator.valuesOverride`           | Custom values to override                                                                                                                                                         | `{}`    |
+| `addons.fluxcd.enabled`                       | Enable FluxCD                                                                                                                                                                     | `false` |
+| `addons.fluxcd.valuesOverride`                | Custom values to override                                                                                                                                                         | `{}`    |
+| `addons.monitoringAgents.enabled`             | Enable monitoring agents (Fluent Bit and VMAgents) to send logs and metrics. If tenant monitoring is enabled, data is sent to tenant storage; otherwise, it goes to root storage. | `false` |
+| `addons.monitoringAgents.valuesOverride`      | Custom values to override                                                                                                                                                         | `{}`    |
+| `addons.verticalPodAutoscaler.valuesOverride` | Custom values to override                                                                                                                                                         | `{}`    |
+
+### Kubernetes Control Plane Configuration
+
+| Name                                               | Description                                                                  | Value   |
+| -------------------------------------------------- | ---------------------------------------------------------------------------- | ------- |
+| `controlPlane.apiServer.resources`                 | Explicit CPU/memory resource requests and limits for the API server.         | `{}`    |
+| `controlPlane.apiServer.resourcesPreset`           | Use a common resources preset when `resources` is not set explicitly.        | `small` |
+| `controlPlane.controllerManager.resources`         | Explicit CPU/memory resource requests and limits for the controller manager. | `{}`    |
+| `controlPlane.controllerManager.resourcesPreset`   | Use a common resources preset when `resources` is not set explicitly.        | `micro` |
+| `controlPlane.scheduler.resources`                 | Explicit CPU/memory resource requests and limits for the scheduler.          | `{}`    |
+| `controlPlane.scheduler.resourcesPreset`           | Use a common resources preset when `resources` is not set explicitly.        | `micro` |
+| `controlPlane.konnectivity.server.resources`       | Explicit CPU/memory resource requests and limits for the Konnectivity.       | `{}`    |
+| `controlPlane.konnectivity.server.resourcesPreset` | Use a common resources preset when `resources` is not set explicitly.        | `micro` |
+
+In production environments, it's recommended to set `resources` explicitly.
+Example of `controlPlane.*.resources`:
+
+```yaml
+resources:
+  limits:
+    cpu: 4000m
+    memory: 4Gi
+  requests:
+    cpu: 100m
+    memory: 512Mi
 ```
 
-# Series
+Allowed values for `controlPlane.*.resourcesPreset` are `none`, `nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`.
+This value is ignored if the corresponding `resources` value is set. 
 
-<!-- source: https://github.com/kubevirt/common-instancetypes/blob/main/README.md -->
+## Resources Reference
 
-.                           |  U  |  O  |  CX  |  M  |  RT
-----------------------------|-----|-----|------|-----|------
-*Has GPUs*                  |     |     |      |     |
-*Hugepages*                 |     |     |  ✓   |  ✓  |  ✓
-*Overcommitted Memory*      |     |  ✓  |      |     |
-*Dedicated CPU*             |     |     |  ✓   |     |  ✓
-*Burstable CPU performance* |  ✓  |  ✓  |      |  ✓  |
-*Isolated emulator threads* |     |     |  ✓   |     |  ✓
-*vNUMA*                     |     |     |  ✓   |     |  ✓
-*vCPU-To-Memory Ratio*      | 1:4 | 1:4 |  1:2 | 1:8 | 1:4
+### instanceType Resources
 
+The following instanceType resources are provided by Cozystack:
 
-## U Series
+| Name          | vCPUs | Memory |
+|---------------|-------|--------|
+| `cx1.2xlarge` | 8     | 16Gi   |
+| `cx1.4xlarge` | 16    | 32Gi   |
+| `cx1.8xlarge` | 32    | 64Gi   |
+| `cx1.large`   | 2     | 4Gi    |
+| `cx1.medium`  | 1     | 2Gi    |
+| `cx1.xlarge`  | 4     | 8Gi    |
+| `gn1.2xlarge` | 8     | 32Gi   |
+| `gn1.4xlarge` | 16    | 64Gi   |
+| `gn1.8xlarge` | 32    | 128Gi  |
+| `gn1.xlarge`  | 4     | 16Gi   |
+| `m1.2xlarge`  | 8     | 64Gi   |
+| `m1.4xlarge`  | 16    | 128Gi  |
+| `m1.8xlarge`  | 32    | 256Gi  |
+| `m1.large`    | 2     | 16Gi   |
+| `m1.xlarge`   | 4     | 32Gi   |
+| `n1.2xlarge`  | 16    | 32Gi   |
+| `n1.4xlarge`  | 32    | 64Gi   |
+| `n1.8xlarge`  | 64    | 128Gi  |
+| `n1.large`    | 4     | 8Gi    |
+| `n1.medium`   | 4     | 4Gi    |
+| `n1.xlarge`   | 8     | 16Gi   |
+| `o1.2xlarge`  | 8     | 32Gi   |
+| `o1.4xlarge`  | 16    | 64Gi   |
+| `o1.8xlarge`  | 32    | 128Gi  |
+| `o1.large`    | 2     | 8Gi    |
+| `o1.medium`   | 1     | 4Gi    |
+| `o1.micro`    | 1     | 1Gi    |
+| `o1.nano`     | 1     | 512Mi  |
+| `o1.small`    | 1     | 2Gi    |
+| `o1.xlarge`   | 4     | 16Gi   |
+| `rt1.2xlarge` | 8     | 32Gi   |
+| `rt1.4xlarge` | 16    | 64Gi   |
+| `rt1.8xlarge` | 32    | 128Gi  |
+| `rt1.large`   | 2     | 8Gi    |
+| `rt1.medium`  | 1     | 4Gi    |
+| `rt1.micro`   | 1     | 1Gi    |
+| `rt1.small`   | 1     | 2Gi    |
+| `rt1.xlarge`  | 4     | 16Gi   |
+| `u1.2xlarge`  | 8     | 32Gi   |
+| `u1.2xmedium` | 2     | 4Gi    |
+| `u1.4xlarge`  | 16    | 64Gi   |
+| `u1.8xlarge`  | 32    | 128Gi  |
+| `u1.large`    | 2     | 8Gi    |
+| `u1.medium`   | 1     | 4Gi    |
+| `u1.micro`    | 1     | 1Gi    |
+| `u1.nano`     | 1     | 512Mi  |
+| `u1.small`    | 1     | 2Gi    |
+| `u1.xlarge`   | 4     | 16Gi   |
+
+### U Series: Universal
 
 The U Series is quite neutral and provides resources for
 general purpose applications.
@@ -54,7 +205,7 @@ attitude towards workloads.
 VMs of instance types will share physical CPU cores on a
 time-slice basis with other VMs.
 
-### U Series Characteristics
+#### U Series Characteristics
 
 Specific characteristics of this series are:
 - *Burstable CPU performance* - The workload has a baseline compute
@@ -63,14 +214,14 @@ Specific characteristics of this series are:
 - *vCPU-To-Memory Ratio (1:4)* - A vCPU-to-Memory ratio of 1:4, for less
   noise per node.
 
-## O Series
+### O Series: Overcommitted
 
 The O Series is based on the U Series, with the only difference
 being that memory is overcommitted.
 
 *O* is the abbreviation for "Overcommitted".
 
-### UO Series Characteristics
+#### O Series Characteristics
 
 Specific characteristics of this series are:
 - *Burstable CPU performance* - The workload has a baseline compute
@@ -81,7 +232,7 @@ Specific characteristics of this series are:
 - *vCPU-To-Memory Ratio (1:4)* - A vCPU-to-Memory ratio of 1:4, for less
   noise per node.
 
-## CX Series
+### CX Series: Compute Exclusive
 
 The CX Series provides exclusive compute resources for compute
 intensive applications.
@@ -95,7 +246,7 @@ the IO threading from cores dedicated to the workload.
 In addition, in this series, the NUMA topology of the used
 cores is provided to the VM.
 
-### CX Series Characteristics
+#### CX Series Characteristics
 
 Specific characteristics of this series are:
 - *Hugepages* - Hugepages are used in order to improve memory
@@ -110,14 +261,14 @@ Specific characteristics of this series are:
   optimize guest sided cache utilization.
 - *vCPU-To-Memory Ratio (1:2)* - A vCPU-to-Memory ratio of 1:2.
 
-## M Series
+### M Series: Memory
 
 The M Series provides resources for memory intensive
 applications.
 
 *M* is the abbreviation of "Memory".
 
-### M Series Characteristics
+#### M Series Characteristics
 
 Specific characteristics of this series are:
 - *Hugepages* - Hugepages are used in order to improve memory
@@ -128,7 +279,7 @@ Specific characteristics of this series are:
 - *vCPU-To-Memory Ratio (1:8)* - A vCPU-to-Memory ratio of 1:8, for much
   less noise per node.
 
-## RT Series
+### RT Series: RealTime
 
 The RT Series provides resources for realtime applications, like Oslat.
 
@@ -137,7 +288,7 @@ The RT Series provides resources for realtime applications, like Oslat.
 This series of instance types requires nodes capable of running
 realtime applications.
 
-### RT Series Characteristics
+#### RT Series Characteristics
 
 Specific characteristics of this series are:
 - *Hugepages* - Hugepages are used in order to improve memory
@@ -151,57 +302,3 @@ Specific characteristics of this series are:
 - *vCPU-To-Memory Ratio (1:4)* - A vCPU-to-Memory ratio of 1:4 starting from
   the medium size.
 
-## Resources
-
-The following instancetype resources are provided by Cozystack:
-
-Name | vCPUs | Memory
------|-------|-------
-cx1.2xlarge  |  8  |  16Gi
-cx1.4xlarge  |  16  |  32Gi
-cx1.8xlarge  |  32  |  64Gi
-cx1.large  |  2  |  4Gi
-cx1.medium  |  1  |  2Gi
-cx1.xlarge  |  4  |  8Gi
-gn1.2xlarge  |  8  |  32Gi
-gn1.4xlarge  |  16  |  64Gi
-gn1.8xlarge  |  32  |  128Gi
-gn1.xlarge  |  4  |  16Gi
-m1.2xlarge  |  8  |  64Gi
-m1.4xlarge  |  16  |  128Gi
-m1.8xlarge  |  32  |  256Gi
-m1.large  |  2  |  16Gi
-m1.xlarge  |  4  |  32Gi
-n1.2xlarge  |  16  |  32Gi
-n1.4xlarge  |  32  |  64Gi
-n1.8xlarge  |  64  |  128Gi
-n1.large  |  4  |  8Gi
-n1.medium  |  4  |  4Gi
-n1.xlarge  |  8  |  16Gi
-o1.2xlarge  |  8  |  32Gi
-o1.4xlarge  |  16  |  64Gi
-o1.8xlarge  |  32  |  128Gi
-o1.large  |  2  |  8Gi
-o1.medium  |  1  |  4Gi
-o1.micro  |  1  |  1Gi
-o1.nano  |  1  |  512Mi
-o1.small  |  1  |  2Gi
-o1.xlarge  |  4  |  16Gi
-rt1.2xlarge  |  8  |  32Gi
-rt1.4xlarge  |  16  |  64Gi
-rt1.8xlarge  |  32  |  128Gi
-rt1.large  |  2  |  8Gi
-rt1.medium  |  1  |  4Gi
-rt1.micro  |  1  |  1Gi
-rt1.small  |  1  |  2Gi
-rt1.xlarge  |  4  |  16Gi
-u1.2xlarge  |  8  |  32Gi
-u1.2xmedium  |  2  |  4Gi
-u1.4xlarge  |  16  |  64Gi
-u1.8xlarge  |  32  |  128Gi
-u1.large  |  2  |  8Gi
-u1.medium  |  1  |  4Gi
-u1.micro  |  1  |  1Gi
-u1.nano  |  1  |  512Mi
-u1.small  |  1  |  2Gi
-u1.xlarge  |  4  |  16Gi

packages/apps/kubernetes/images/cluster-autoscaler.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/cluster-autoscaler:0.15.2@sha256:967e51702102d0dbd97f9847de4159d62681b31eb606322d2c29755393c2236e
+ghcr.io/cozystack/cozystack/cluster-autoscaler:0.23.1@sha256:7315850634728a5864a3de3150c12f0e1454f3f1ce33cdf21a278f57611dd5e9

packages/apps/kubernetes/images/cluster-autoscaler/Dockerfile

@@ -1,7 +1,14 @@
 # Source: https://raw.githubusercontent.com/kubernetes/autoscaler/refs/heads/master/cluster-autoscaler/Dockerfile.amd64
 ARG builder_image=docker.io/library/golang:1.23.4
-ARG BASEIMAGE=gcr.io/distroless/static:nonroot-amd64
+ARG BASEIMAGE=gcr.io/distroless/static:nonroot-${TARGETARCH}
+
 FROM ${builder_image} AS builder
+
+ARG TARGETOS
+ARG TARGETARCH
+ENV GOOS=$TARGETOS
+ENV GOARCH=$TARGETARCH
+
 RUN git clone https://github.com/kubernetes/autoscaler /src/autoscaler \
  && cd /src/autoscaler/cluster-autoscaler \
  && git checkout cluster-autoscaler-1.32.0
@@ -14,6 +21,8 @@ RUN make build
 FROM $BASEIMAGE
 LABEL maintainer="Marcin Wielgus <mwielgus@google.com>"
 
-COPY --from=builder /src/autoscaler/cluster-autoscaler/cluster-autoscaler-amd64 /cluster-autoscaler
+ARG TARGETARCH
+
+COPY --from=builder /src/autoscaler/cluster-autoscaler/cluster-autoscaler-${TARGETARCH} /cluster-autoscaler
 WORKDIR /
 CMD ["/cluster-autoscaler"]

packages/apps/kubernetes/images/kubevirt-cloud-provider.tag

@@ -1 +1 @@
-ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:0.15.2@sha256:5e054eae6274963b6e84f87bf3330c94325103c6407b08bfb1189da721333b5c
+ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:0.23.1@sha256:6962bdf51ab2ff40b420b9cff7c850aeea02187da2a65a67f10e0471744649d7

packages/apps/kubernetes/images/kubevirt-cloud-provider/Dockerfile

@@ -1,21 +1,25 @@
 # Source: https://github.com/kubevirt/cloud-provider-kubevirt/blob/main/build/images/kubevirt-cloud-controller-manager/Dockerfile
-FROM --platform=linux/amd64 golang:1.20.6 AS builder
+FROM golang:1.20.6 AS builder
+
+ARG TARGETOS
+ARG TARGETARCH
+ENV GOOS=$TARGETOS
+ENV GOARCH=$TARGETARCH
 
 RUN git clone https://github.com/kubevirt/cloud-provider-kubevirt /go/src/kubevirt.io/cloud-provider-kubevirt \
  && cd /go/src/kubevirt.io/cloud-provider-kubevirt \
- && git checkout da9e0cf
+ && git checkout 443a1fe
 
 WORKDIR /go/src/kubevirt.io/cloud-provider-kubevirt
 
 # see: https://github.com/kubevirt/cloud-provider-kubevirt/pull/335
-# see: https://github.com/kubevirt/cloud-provider-kubevirt/pull/336
 ADD patches /patches
 RUN git apply /patches/*.diff
 RUN go get 'k8s.io/endpointslice/util@v0.28' 'k8s.io/apiserver@v0.28'
 RUN go mod tidy
 RUN go mod vendor
 
-RUN	CGO_ENABLED=0 GOOS=linux go build -mod=vendor -ldflags="-s -w" -o bin/kubevirt-cloud-controller-manager ./cmd/kubevirt-cloud-controller-manager
+RUN CGO_ENABLED=0 go build -mod=vendor -ldflags="-s -w" -o bin/kubevirt-cloud-controller-manager ./cmd/kubevirt-cloud-controller-manager
 
 FROM registry.access.redhat.com/ubi9/ubi-micro
 COPY --from=builder /go/src/kubevirt.io/cloud-provider-kubevirt/bin/kubevirt-cloud-controller-manager /bin/kubevirt-cloud-controller-manager

packages/apps/kubernetes/images/kubevirt-cloud-provider/patches/335.diff

@@ -1,20 +0,0 @@
-diff --git a/pkg/controller/kubevirteps/kubevirteps_controller.go b/pkg/controller/kubevirteps/kubevirteps_controller.go
-index a3c1aa33..95c31438 100644
---- a/pkg/controller/kubevirteps/kubevirteps_controller.go
-+++ b/pkg/controller/kubevirteps/kubevirteps_controller.go
-@@ -412,11 +412,11 @@ func (c *Controller) reconcileByAddressType(service *v1.Service, tenantSlices []
- 	// Create the desired port configuration
- 	var desiredPorts []discovery.EndpointPort
- 
--	for _, port := range service.Spec.Ports {
-+	for i := range service.Spec.Ports {
- 		desiredPorts = append(desiredPorts, discovery.EndpointPort{
--			Port:     &port.TargetPort.IntVal,
--			Protocol: &port.Protocol,
--			Name:     &port.Name,
-+			Port:     &service.Spec.Ports[i].TargetPort.IntVal,
-+			Protocol: &service.Spec.Ports[i].Protocol,
-+			Name:     &service.Spec.Ports[i].Name,
- 		})
- 	}
- 

packages/apps/kubernetes/images/kubevirt-cloud-provider/patches/336.diff

@@ -1,129 +0,0 @@
-diff --git a/pkg/controller/kubevirteps/kubevirteps_controller.go b/pkg/controller/kubevirteps/kubevirteps_controller.go
-index a3c1aa33..6f6e3d32 100644
---- a/pkg/controller/kubevirteps/kubevirteps_controller.go
-+++ b/pkg/controller/kubevirteps/kubevirteps_controller.go
-@@ -108,32 +108,24 @@ func newRequest(reqType ReqType, obj interface{}, oldObj interface{}) *Request {
- }
- 
- func (c *Controller) Init() error {
--
--	// Act on events from Services on the infra cluster. These are created by the EnsureLoadBalancer function.
--	// We need to watch for these events so that we can update the EndpointSlices in the infra cluster accordingly.
-+	// Existing Service event handlers...
- 	_, err := c.infraFactory.Core().V1().Services().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
- 		AddFunc: func(obj interface{}) {
--			// cast obj to Service
- 			svc := obj.(*v1.Service)
--			// Only act on Services of type LoadBalancer
- 			if svc.Spec.Type == v1.ServiceTypeLoadBalancer {
- 				klog.Infof("Service added: %v/%v", svc.Namespace, svc.Name)
- 				c.queue.Add(newRequest(AddReq, obj, nil))
- 			}
- 		},
- 		UpdateFunc: func(oldObj, newObj interface{}) {
--			// cast obj to Service
- 			newSvc := newObj.(*v1.Service)
--			// Only act on Services of type LoadBalancer
- 			if newSvc.Spec.Type == v1.ServiceTypeLoadBalancer {
- 				klog.Infof("Service updated: %v/%v", newSvc.Namespace, newSvc.Name)
- 				c.queue.Add(newRequest(UpdateReq, newObj, oldObj))
- 			}
- 		},
- 		DeleteFunc: func(obj interface{}) {
--			// cast obj to Service
- 			svc := obj.(*v1.Service)
--			// Only act on Services of type LoadBalancer
- 			if svc.Spec.Type == v1.ServiceTypeLoadBalancer {
- 				klog.Infof("Service deleted: %v/%v", svc.Namespace, svc.Name)
- 				c.queue.Add(newRequest(DeleteReq, obj, nil))
-@@ -144,7 +136,7 @@ func (c *Controller) Init() error {
- 		return err
- 	}
- 
--	// Monitor endpoint slices that we are interested in based on known services in the infra cluster
-+	// Existing EndpointSlice event handlers in tenant cluster...
- 	_, err = c.tenantFactory.Discovery().V1().EndpointSlices().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
- 		AddFunc: func(obj interface{}) {
- 			eps := obj.(*discovery.EndpointSlice)
-@@ -194,10 +186,80 @@ func (c *Controller) Init() error {
- 		return err
- 	}
- 
--	//TODO: Add informer for EndpointSlices in the infra cluster to watch for (unwanted) changes
-+	// Add an informer for EndpointSlices in the infra cluster
-+	_, err = c.infraFactory.Discovery().V1().EndpointSlices().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-+		AddFunc: func(obj interface{}) {
-+			eps := obj.(*discovery.EndpointSlice)
-+			if c.managedByController(eps) {
-+				svc, svcErr := c.getInfraServiceForEPS(context.TODO(), eps)
-+				if svcErr != nil {
-+					klog.Errorf("Failed to get infra Service for EndpointSlice %s/%s: %v", eps.Namespace, eps.Name, svcErr)
-+					return
-+				}
-+				if svc != nil {
-+					klog.Infof("Infra EndpointSlice added: %v/%v, requeuing Service: %v/%v", eps.Namespace, eps.Name, svc.Namespace, svc.Name)
-+					c.queue.Add(newRequest(AddReq, svc, nil))
-+				}
-+			}
-+		},
-+		UpdateFunc: func(oldObj, newObj interface{}) {
-+			eps := newObj.(*discovery.EndpointSlice)
-+			if c.managedByController(eps) {
-+				svc, svcErr := c.getInfraServiceForEPS(context.TODO(), eps)
-+				if svcErr != nil {
-+					klog.Errorf("Failed to get infra Service for EndpointSlice %s/%s: %v", eps.Namespace, eps.Name, svcErr)
-+					return
-+				}
-+				if svc != nil {
-+					klog.Infof("Infra EndpointSlice updated: %v/%v, requeuing Service: %v/%v", eps.Namespace, eps.Name, svc.Namespace, svc.Name)
-+					c.queue.Add(newRequest(UpdateReq, svc, nil))
-+				}
-+			}
-+		},
-+		DeleteFunc: func(obj interface{}) {
-+			eps := obj.(*discovery.EndpointSlice)
-+			if c.managedByController(eps) {
-+				svc, svcErr := c.getInfraServiceForEPS(context.TODO(), eps)
-+				if svcErr != nil {
-+					klog.Errorf("Failed to get infra Service for EndpointSlice %s/%s on delete: %v", eps.Namespace, eps.Name, svcErr)
-+					return
-+				}
-+				if svc != nil {
-+					klog.Infof("Infra EndpointSlice deleted: %v/%v, requeuing Service: %v/%v", eps.Namespace, eps.Name, svc.Namespace, svc.Name)
-+					c.queue.Add(newRequest(DeleteReq, svc, nil))
-+				}
-+			}
-+		},
-+	})
-+	if err != nil {
-+		return err
-+	}
-+
- 	return nil
- }
- 
-+// getInfraServiceForEPS returns the Service in the infra cluster associated with the given EndpointSlice.
-+// It does this by reading the "kubernetes.io/service-name" label from the EndpointSlice, which should correspond
-+// to the Service name. If not found or if the Service doesn't exist, it returns nil.
-+func (c *Controller) getInfraServiceForEPS(ctx context.Context, eps *discovery.EndpointSlice) (*v1.Service, error) {
-+	svcName := eps.Labels[discovery.LabelServiceName]
-+	if svcName == "" {
-+		// No service name label found, can't determine infra service.
-+		return nil, nil
-+	}
-+
-+	svc, err := c.infraClient.CoreV1().Services(c.infraNamespace).Get(ctx, svcName, metav1.GetOptions{})
-+	if err != nil {
-+		if k8serrors.IsNotFound(err) {
-+			// Service doesn't exist
-+			return nil, nil
-+		}
-+		return nil, err
-+	}
-+
-+	return svc, nil
-+}
-+
- // Run starts an asynchronous loop that monitors and updates GKENetworkParamSet in the cluster.
- func (c *Controller) Run(numWorkers int, stopCh <-chan struct{}, controllerManagerMetrics *controllersmetrics.ControllerManagerMetrics) {
- 	defer utilruntime.HandleCrash()

packages/apps/kubernetes/images/kubevirt-cloud-provider/patches/341.diff

@@ -0,0 +1,689 @@
+diff --git a/.golangci.yml b/.golangci.yml
+index cf72a41a2..1c9237e83 100644
+--- a/.golangci.yml
++++ b/.golangci.yml
+@@ -122,3 +122,9 @@ linters:
+   # - testpackage
+   # - revive
+   # - wsl
++issues:
++  exclude-rules:
++    - filename: "kubevirteps_controller_test.go"
++      linters:
++        - govet
++      text: "declaration of \"err\" shadows"
+diff --git a/cmd/kubevirt-cloud-controller-manager/kubevirteps.go b/cmd/kubevirt-cloud-controller-manager/kubevirteps.go
+index 74166b5d9..4e744f8de 100644
+--- a/cmd/kubevirt-cloud-controller-manager/kubevirteps.go
++++ b/cmd/kubevirt-cloud-controller-manager/kubevirteps.go
+@@ -101,7 +101,18 @@ func startKubevirtCloudController(
+ 
+ 	klog.Infof("Setting up kubevirtEPSController")
+ 
+-	kubevirtEPSController := kubevirteps.NewKubevirtEPSController(tenantClient, infraClient, infraDynamic, kubevirtCloud.Namespace())
++	clusterName := ccmConfig.ComponentConfig.KubeCloudShared.ClusterName
++	if clusterName == "" {
++		klog.Fatalf("Required flag --cluster-name is missing")
++	}
++
++	kubevirtEPSController := kubevirteps.NewKubevirtEPSController(
++		tenantClient,
++		infraClient,
++		infraDynamic,
++		kubevirtCloud.Namespace(),
++		clusterName,
++	)
+ 
+ 	klog.Infof("Initializing kubevirtEPSController")
+ 
+diff --git a/pkg/controller/kubevirteps/kubevirteps_controller.go b/pkg/controller/kubevirteps/kubevirteps_controller.go
+index 6f6e3d322..b56882c12 100644
+--- a/pkg/controller/kubevirteps/kubevirteps_controller.go
++++ b/pkg/controller/kubevirteps/kubevirteps_controller.go
+@@ -54,10 +54,10 @@ type Controller struct {
+ 	infraDynamic dynamic.Interface
+ 	infraFactory informers.SharedInformerFactory
+ 
+-	infraNamespace string
+-	queue          workqueue.RateLimitingInterface
+-	maxRetries     int
+-
++	infraNamespace       string
++	clusterName          string
++	queue                workqueue.RateLimitingInterface
++	maxRetries           int
+ 	maxEndPointsPerSlice int
+ }
+ 
+@@ -65,8 +65,9 @@ func NewKubevirtEPSController(
+ 	tenantClient kubernetes.Interface,
+ 	infraClient kubernetes.Interface,
+ 	infraDynamic dynamic.Interface,
+-	infraNamespace string) *Controller {
+-
++	infraNamespace string,
++	clusterName string,
++) *Controller {
+ 	tenantFactory := informers.NewSharedInformerFactory(tenantClient, 0)
+ 	infraFactory := informers.NewSharedInformerFactoryWithOptions(infraClient, 0, informers.WithNamespace(infraNamespace))
+ 	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
+@@ -79,6 +80,7 @@ func NewKubevirtEPSController(
+ 		infraDynamic:         infraDynamic,
+ 		infraFactory:         infraFactory,
+ 		infraNamespace:       infraNamespace,
++		clusterName:          clusterName,
+ 		queue:                queue,
+ 		maxRetries:           25,
+ 		maxEndPointsPerSlice: 100,
+@@ -320,22 +322,30 @@ func (c *Controller) processNextItem(ctx context.Context) bool {
+ 
+ // getInfraServiceFromTenantEPS returns the Service in the infra cluster that is associated with the given tenant endpoint slice.
+ func (c *Controller) getInfraServiceFromTenantEPS(ctx context.Context, slice *discovery.EndpointSlice) (*v1.Service, error) {
+-	infraServices, err := c.infraClient.CoreV1().Services(c.infraNamespace).List(ctx,
+-		metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s=%s", kubevirt.TenantServiceNameLabelKey, slice.Labels["kubernetes.io/service-name"],
+-			kubevirt.TenantServiceNamespaceLabelKey, slice.Namespace)})
++	tenantServiceName := slice.Labels[discovery.LabelServiceName]
++	tenantServiceNamespace := slice.Namespace
++
++	labelSelector := fmt.Sprintf(
++		"%s=%s,%s=%s,%s=%s",
++		kubevirt.TenantServiceNameLabelKey, tenantServiceName,
++		kubevirt.TenantServiceNamespaceLabelKey, tenantServiceNamespace,
++		kubevirt.TenantClusterNameLabelKey, c.clusterName,
++	)
++
++	svcList, err := c.infraClient.CoreV1().Services(c.infraNamespace).List(ctx, metav1.ListOptions{
++		LabelSelector: labelSelector,
++	})
+ 	if err != nil {
+-		klog.Errorf("Failed to get Service in Infra for EndpointSlice %s in namespace %s: %v", slice.Name, slice.Namespace, err)
++		klog.Errorf("Failed to get Service in Infra for EndpointSlice %s in namespace %s: %v", slice.Name, tenantServiceNamespace, err)
+ 		return nil, err
+ 	}
+-	if len(infraServices.Items) > 1 {
+-		// This should never be possible, only one service should exist for a given tenant endpoint slice
+-		klog.Errorf("Multiple services found for tenant endpoint slice %s in namespace %s", slice.Name, slice.Namespace)
++	if len(svcList.Items) > 1 {
++		klog.Errorf("Multiple services found for tenant endpoint slice %s in namespace %s", slice.Name, tenantServiceNamespace)
+ 		return nil, errors.New("multiple services found for tenant endpoint slice")
+ 	}
+-	if len(infraServices.Items) == 1 {
+-		return &infraServices.Items[0], nil
++	if len(svcList.Items) == 1 {
++		return &svcList.Items[0], nil
+ 	}
+-	// No service found, possible if service is deleted.
+ 	return nil, nil
+ }
+ 
+@@ -363,16 +373,27 @@ func (c *Controller) getTenantEPSFromInfraService(ctx context.Context, svc *v1.S
+ // getInfraEPSFromInfraService returns the EndpointSlices in the infra cluster that are associated with the given infra service.
+ func (c *Controller) getInfraEPSFromInfraService(ctx context.Context, svc *v1.Service) ([]*discovery.EndpointSlice, error) {
+ 	var infraEPSSlices []*discovery.EndpointSlice
+-	klog.Infof("Searching for endpoints on infra cluster for service %s in namespace %s.", svc.Name, svc.Namespace)
+-	result, err := c.infraClient.DiscoveryV1().EndpointSlices(svc.Namespace).List(ctx,
+-		metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", discovery.LabelServiceName, svc.Name)})
++
++	klog.Infof("Searching for EndpointSlices in infra cluster for service %s/%s", svc.Namespace, svc.Name)
++
++	labelSelector := fmt.Sprintf(
++		"%s=%s,%s=%s",
++		discovery.LabelServiceName, svc.Name,
++		kubevirt.TenantClusterNameLabelKey, c.clusterName,
++	)
++
++	result, err := c.infraClient.DiscoveryV1().EndpointSlices(svc.Namespace).List(ctx, metav1.ListOptions{
++		LabelSelector: labelSelector,
++	})
+ 	if err != nil {
+ 		klog.Errorf("Failed to get EndpointSlices for Service %s in namespace %s: %v", svc.Name, svc.Namespace, err)
+ 		return nil, err
+ 	}
++
+ 	for _, eps := range result.Items {
+ 		infraEPSSlices = append(infraEPSSlices, &eps)
+ 	}
++
+ 	return infraEPSSlices, nil
+ }
+ 
+@@ -382,74 +403,117 @@ func (c *Controller) reconcile(ctx context.Context, r *Request) error {
+ 		return errors.New("could not cast object to service")
+ 	}
+ 
++	// Skip services not managed by this controller (missing required labels)
+ 	if service.Labels[kubevirt.TenantServiceNameLabelKey] == "" ||
+ 		service.Labels[kubevirt.TenantServiceNamespaceLabelKey] == "" ||
+ 		service.Labels[kubevirt.TenantClusterNameLabelKey] == "" {
+-		klog.Infof("This LoadBalancer Service: %s is not managed by the %s. Skipping.", service.Name, ControllerName)
++		klog.Infof("Service %s is not managed by this controller. Skipping.", service.Name)
++		return nil
++	}
++
++	// Skip services for other clusters
++	if service.Labels[kubevirt.TenantClusterNameLabelKey] != c.clusterName {
++		klog.Infof("Skipping Service %s: cluster label %q doesn't match our clusterName %q", service.Name, service.Labels[kubevirt.TenantClusterNameLabelKey], c.clusterName)
+ 		return nil
+ 	}
++
+ 	klog.Infof("Reconciling: %v", service.Name)
+ 
++	/*
++	   1) Check if Service in the infra cluster is actually present.
++	      If it's not found, mark it as 'deleted' so that we don't create new slices.
++	*/
+ 	serviceDeleted := false
+-	svc, err := c.infraFactory.Core().V1().Services().Lister().Services(c.infraNamespace).Get(service.Name)
++	infraSvc, err := c.infraFactory.Core().V1().Services().Lister().Services(c.infraNamespace).Get(service.Name)
+ 	if err != nil {
+-		klog.Infof("Service %s in namespace %s is deleted.", service.Name, service.Namespace)
++		// The Service is not present in the infra lister => treat as deleted
++		klog.Infof("Service %s in namespace %s is deleted (or not found).", service.Name, service.Namespace)
+ 		serviceDeleted = true
+ 	} else {
+-		service = svc
++		// Use the actual object from the lister, so we have the latest state
++		service = infraSvc
+ 	}
+ 
++	/*
++	   2) Get all existing EndpointSlices in the infra cluster that belong to this LB Service.
++	      We'll decide which of them should be updated or deleted.
++	*/
+ 	infraExistingEpSlices, err := c.getInfraEPSFromInfraService(ctx, service)
+ 	if err != nil {
+ 		return err
+ 	}
+ 
+-	// At this point we have the current state of the 3 main objects we are interested in:
+-	// 1. The Service in the infra cluster, the one created by the KubevirtCloudController.
+-	// 2. The EndpointSlices in the tenant cluster, created for the tenant cluster's Service.
+-	// 3. The EndpointSlices in the infra cluster, managed by this controller.
+-
+ 	slicesToDelete := []*discovery.EndpointSlice{}
+ 	slicesByAddressType := make(map[discovery.AddressType][]*discovery.EndpointSlice)
+ 
++	// For example, if the service is single-stack IPv4 => only AddressTypeIPv4
++	// or if dual-stack => IPv4 and IPv6, etc.
+ 	serviceSupportedAddressesTypes := getAddressTypesForService(service)
+-	// If the services switched to a different address type, we need to delete the old ones, because it's immutable.
+-	// If the services switched to a different externalTrafficPolicy, we need to delete the old ones.
++
++	/*
++	   3) Determine which slices to delete, and which to pass on to the normal
++	      "reconcileByAddressType" logic.
++
++	      - If 'serviceDeleted' is true OR service.Spec.Selector != nil, we remove them.
++	      - Also, if the slice's address type is unsupported by the Service, we remove it.
++	*/
+ 	for _, eps := range infraExistingEpSlices {
+-		if service.Spec.Selector != nil || serviceDeleted {
+-			klog.Infof("Added for deletion EndpointSlice %s in namespace %s because it has a selector", eps.Name, eps.Namespace)
+-			// to be sure we don't delete any slice that is not managed by us
++		// If service is deleted or has a non-nil selector => remove slices
++		if serviceDeleted || service.Spec.Selector != nil {
++			/*
++			   Only remove if it is clearly labeled as managed by us:
++			   we do not want to accidentally remove slices that are not
++			   created by this controller.
++			*/
+ 			if c.managedByController(eps) {
++				klog.Infof("Added for deletion EndpointSlice %s in namespace %s because service is deleted or has a selector",
++					eps.Name, eps.Namespace)
+ 				slicesToDelete = append(slicesToDelete, eps)
+ 			}
+ 			continue
+ 		}
++
++		// If the Service does not support this slice's AddressType => remove
+ 		if !serviceSupportedAddressesTypes.Has(eps.AddressType) {
+-			klog.Infof("Added for deletion EndpointSlice %s in namespace %s because it has an unsupported address type: %v", eps.Name, eps.Namespace, eps.AddressType)
++			klog.Infof("Added for deletion EndpointSlice %s in namespace %s because it has an unsupported address type: %v",
++				eps.Name, eps.Namespace, eps.AddressType)
+ 			slicesToDelete = append(slicesToDelete, eps)
+ 			continue
+ 		}
++
++		/*
++		   Otherwise, this slice is potentially still valid for the given AddressType,
++		   we'll send it to reconcileByAddressType for final merging and updates.
++		*/
+ 		slicesByAddressType[eps.AddressType] = append(slicesByAddressType[eps.AddressType], eps)
+ 	}
+ 
+-	if !serviceDeleted {
+-		// Get tenant's endpoint slices for this service
++	/*
++	   4) If the Service was NOT deleted and has NO selector (i.e., it's a "no-selector" LB Service),
++	      we proceed to handle creation and updates. That means:
++	      - Gather Tenant's EndpointSlices
++	      - Reconcile them by each AddressType
++	*/
++	if !serviceDeleted && service.Spec.Selector == nil {
+ 		tenantEpSlices, err := c.getTenantEPSFromInfraService(ctx, service)
+ 		if err != nil {
+ 			return err
+ 		}
+ 
+-		// Reconcile the EndpointSlices for each address type e.g. ipv4, ipv6
++		// For each addressType (ipv4, ipv6, etc.) reconcile the infra slices
+ 		for addressType := range serviceSupportedAddressesTypes {
+ 			existingSlices := slicesByAddressType[addressType]
+-			err := c.reconcileByAddressType(service, tenantEpSlices, existingSlices, addressType)
+-			if err != nil {
++			if err := c.reconcileByAddressType(service, tenantEpSlices, existingSlices, addressType); err != nil {
+ 				return err
+ 			}
+ 		}
+ 	}
+ 
+-	// Delete the EndpointSlices that are no longer needed
++	/*
++	   5) Perform the actual deletion of all slices we flagged.
++	      In many cases (serviceDeleted or .Spec.Selector != nil),
++	      we end up with only "delete" actions and no new slice creation.
++	*/
+ 	for _, eps := range slicesToDelete {
+ 		err := c.infraClient.DiscoveryV1().EndpointSlices(eps.Namespace).Delete(context.TODO(), eps.Name, metav1.DeleteOptions{})
+ 		if err != nil {
+@@ -474,11 +538,11 @@ func (c *Controller) reconcileByAddressType(service *v1.Service, tenantSlices []
+ 	// Create the desired port configuration
+ 	var desiredPorts []discovery.EndpointPort
+ 
+-	for _, port := range service.Spec.Ports {
++	for i := range service.Spec.Ports {
+ 		desiredPorts = append(desiredPorts, discovery.EndpointPort{
+-			Port:     &port.TargetPort.IntVal,
+-			Protocol: &port.Protocol,
+-			Name:     &port.Name,
++			Port:     &service.Spec.Ports[i].TargetPort.IntVal,
++			Protocol: &service.Spec.Ports[i].Protocol,
++			Name:     &service.Spec.Ports[i].Name,
+ 		})
+ 	}
+ 
+@@ -588,55 +652,114 @@ func ownedBy(endpointSlice *discovery.EndpointSlice, svc *v1.Service) bool {
+ 	return false
+ }
+ 
+-func (c *Controller) finalize(service *v1.Service, slicesToCreate []*discovery.EndpointSlice, slicesToUpdate []*discovery.EndpointSlice, slicesToDelete []*discovery.EndpointSlice) error {
+-	// If there are slices to delete and slices to create, make them as update
+-	for i := 0; i < len(slicesToDelete); {
++func (c *Controller) finalize(
++	service *v1.Service,
++	slicesToCreate []*discovery.EndpointSlice,
++	slicesToUpdate []*discovery.EndpointSlice,
++	slicesToDelete []*discovery.EndpointSlice,
++) error {
++	/*
++	   We try to turn a "delete + create" pair into a single "update" operation
++	   if the original slice (slicesToDelete[i]) has the same address type as
++	   the first slice in slicesToCreate, and is owned by the same Service.
++
++	   However, we must re-check the lengths of slicesToDelete and slicesToCreate
++	   within the loop to avoid an out-of-bounds index in slicesToCreate.
++	*/
++
++	i := 0
++	for i < len(slicesToDelete) {
++		// If there is nothing to create, break early
+ 		if len(slicesToCreate) == 0 {
+ 			break
+ 		}
+-		if slicesToDelete[i].AddressType == slicesToCreate[0].AddressType && ownedBy(slicesToDelete[i], service) {
+-			slicesToCreate[0].Name = slicesToDelete[i].Name
++
++		sd := slicesToDelete[i]
++		sc := slicesToCreate[0] // We can safely do this now, because len(slicesToCreate) > 0
++
++		// If the address type matches, and the slice is owned by the same Service,
++		// then instead of deleting sd and creating sc, we'll transform it into an update:
++		// we rename sc with sd's name, remove sd from the delete list, remove sc from the create list,
++		// and add sc to the update list.
++		if sd.AddressType == sc.AddressType && ownedBy(sd, service) {
++			sliceToUpdate := sc
++			sliceToUpdate.Name = sd.Name
++
++			// Remove the first element from slicesToCreate
+ 			slicesToCreate = slicesToCreate[1:]
+-			slicesToUpdate = append(slicesToUpdate, slicesToCreate[0])
++
++			// Remove the slice from slicesToDelete
+ 			slicesToDelete = append(slicesToDelete[:i], slicesToDelete[i+1:]...)
++
++			// Now add the renamed slice to the list of slices we want to update
++			slicesToUpdate = append(slicesToUpdate, sliceToUpdate)
++
++			/*
++			   Do not increment i here, because we've just removed an element from
++			   slicesToDelete. The next slice to examine is now at the same index i.
++			*/
+ 		} else {
++			// If they don't match, move on to the next slice in slicesToDelete.
+ 			i++
+ 		}
+ 	}
+ 
+-	// Create the new slices if service is not marked for deletion
++	/*
++	   If the Service is not being deleted, create all remaining slices in slicesToCreate.
++	   (If the Service has a DeletionTimestamp, it means it is going away, so we do not
++	   want to create new EndpointSlices.)
++	*/
+ 	if service.DeletionTimestamp == nil {
+ 		for _, slice := range slicesToCreate {
+-			createdSlice, err := c.infraClient.DiscoveryV1().EndpointSlices(slice.Namespace).Create(context.TODO(), slice, metav1.CreateOptions{})
++			createdSlice, err := c.infraClient.DiscoveryV1().EndpointSlices(slice.Namespace).Create(
++				context.TODO(),
++				slice,
++				metav1.CreateOptions{},
++			)
+ 			if err != nil {
+-				klog.Errorf("Failed to create EndpointSlice %s in namespace %s: %v", slice.Name, slice.Namespace, err)
++				klog.Errorf("Failed to create EndpointSlice %s in namespace %s: %v",
++					slice.Name, slice.Namespace, err)
++				// If the namespace is terminating, it's safe to ignore the error.
+ 				if k8serrors.HasStatusCause(err, v1.NamespaceTerminatingCause) {
+-					return nil
++					continue
+ 				}
+ 				return err
+ 			}
+-			klog.Infof("Created EndpointSlice %s in namespace %s", createdSlice.Name, createdSlice.Namespace)
++			klog.Infof("Created EndpointSlice %s in namespace %s",
++				createdSlice.Name, createdSlice.Namespace)
+ 		}
+ 	}
+ 
+-	// Update slices
++	// Update slices that are in the slicesToUpdate list.
+ 	for _, slice := range slicesToUpdate {
+-		_, err := c.infraClient.DiscoveryV1().EndpointSlices(slice.Namespace).Update(context.TODO(), slice, metav1.UpdateOptions{})
++		_, err := c.infraClient.DiscoveryV1().EndpointSlices(slice.Namespace).Update(
++			context.TODO(),
++			slice,
++			metav1.UpdateOptions{},
++		)
+ 		if err != nil {
+-			klog.Errorf("Failed to update EndpointSlice %s in namespace %s: %v", slice.Name, slice.Namespace, err)
++			klog.Errorf("Failed to update EndpointSlice %s in namespace %s: %v",
++				slice.Name, slice.Namespace, err)
+ 			return err
+ 		}
+-		klog.Infof("Updated EndpointSlice %s in namespace %s", slice.Name, slice.Namespace)
++		klog.Infof("Updated EndpointSlice %s in namespace %s",
++			slice.Name, slice.Namespace)
+ 	}
+ 
+-	// Delete slices
++	// Finally, delete slices that are in slicesToDelete and are no longer needed.
+ 	for _, slice := range slicesToDelete {
+-		err := c.infraClient.DiscoveryV1().EndpointSlices(slice.Namespace).Delete(context.TODO(), slice.Name, metav1.DeleteOptions{})
++		err := c.infraClient.DiscoveryV1().EndpointSlices(slice.Namespace).Delete(
++			context.TODO(),
++			slice.Name,
++			metav1.DeleteOptions{},
++		)
+ 		if err != nil {
+-			klog.Errorf("Failed to delete EndpointSlice %s in namespace %s: %v", slice.Name, slice.Namespace, err)
++			klog.Errorf("Failed to delete EndpointSlice %s in namespace %s: %v",
++				slice.Name, slice.Namespace, err)
+ 			return err
+ 		}
+-		klog.Infof("Deleted EndpointSlice %s in namespace %s", slice.Name, slice.Namespace)
++		klog.Infof("Deleted EndpointSlice %s in namespace %s",
++			slice.Name, slice.Namespace)
+ 	}
+ 
+ 	return nil
+diff --git a/pkg/controller/kubevirteps/kubevirteps_controller_test.go b/pkg/controller/kubevirteps/kubevirteps_controller_test.go
+index 1fb86e25f..14d92d340 100644
+--- a/pkg/controller/kubevirteps/kubevirteps_controller_test.go
++++ b/pkg/controller/kubevirteps/kubevirteps_controller_test.go
+@@ -13,6 +13,7 @@ import (
+ 	"k8s.io/apimachinery/pkg/runtime"
+ 	"k8s.io/apimachinery/pkg/runtime/schema"
+ 	"k8s.io/apimachinery/pkg/util/intstr"
++	"k8s.io/apimachinery/pkg/util/sets"
+ 	dfake "k8s.io/client-go/dynamic/fake"
+ 	"k8s.io/client-go/kubernetes/fake"
+ 	"k8s.io/client-go/testing"
+@@ -189,7 +190,7 @@ func setupTestKubevirtEPSController() *testKubevirtEPSController {
+ 		}: "VirtualMachineInstanceList",
+ 	})
+ 
+-	controller := NewKubevirtEPSController(tenantClient, infraClient, infraDynamic, "test")
++	controller := NewKubevirtEPSController(tenantClient, infraClient, infraDynamic, "test", "test-cluster")
+ 
+ 	err := controller.Init()
+ 	if err != nil {
+@@ -686,5 +687,229 @@ var _ = g.Describe("KubevirtEPSController", g.Ordered, func() {
+ 				return false, err
+ 			}).Should(BeTrue(), "EndpointSlice in infra cluster should be recreated by the controller after deletion")
+ 		})
++
++		g.It("Should correctly handle multiple unique ports in EndpointSlice", func() {
++			// Create a VMI in the infra cluster
++			createAndAssertVMI("worker-0-test", "ip-10-32-5-13", "123.45.67.89")
++
++			// Create an EndpointSlice in the tenant cluster
++			createAndAssertTenantSlice("test-epslice", "tenant-service-name", discoveryv1.AddressTypeIPv4,
++				*createPort("http", 80, v1.ProtocolTCP),
++				[]discoveryv1.Endpoint{*createEndpoint("123.45.67.89", "worker-0-test", true, true, false)})
++
++			// Define multiple ports for the Service
++			servicePorts := []v1.ServicePort{
++				{
++					Name:       "client",
++					Protocol:   v1.ProtocolTCP,
++					Port:       10001,
++					TargetPort: intstr.FromInt(30396),
++					NodePort:   30396,
++				},
++				{
++					Name:       "dashboard",
++					Protocol:   v1.ProtocolTCP,
++					Port:       8265,
++					TargetPort: intstr.FromInt(31003),
++					NodePort:   31003,
++				},
++				{
++					Name:       "metrics",
++					Protocol:   v1.ProtocolTCP,
++					Port:       8080,
++					TargetPort: intstr.FromInt(30452),
++					NodePort:   30452,
++				},
++			}
++
++			createAndAssertInfraServiceLB("infra-multiport-service", "tenant-service-name", "test-cluster",
++				servicePorts[0], v1.ServiceExternalTrafficPolicyLocal)
++
++			svc, err := testVals.infraClient.CoreV1().Services(infraNamespace).Get(context.TODO(), "infra-multiport-service", metav1.GetOptions{})
++			Expect(err).To(BeNil())
++
++			svc.Spec.Ports = servicePorts
++			_, err = testVals.infraClient.CoreV1().Services(infraNamespace).Update(context.TODO(), svc, metav1.UpdateOptions{})
++			Expect(err).To(BeNil())
++
++			var epsListMultiPort *discoveryv1.EndpointSliceList
++
++			Eventually(func() (bool, error) {
++				epsListMultiPort, err = testVals.infraClient.DiscoveryV1().EndpointSlices(infraNamespace).List(context.TODO(), metav1.ListOptions{})
++				if len(epsListMultiPort.Items) != 1 {
++					return false, err
++				}
++
++				createdSlice := epsListMultiPort.Items[0]
++				expectedPortNames := []string{"client", "dashboard", "metrics"}
++				foundPortNames := []string{}
++
++				for _, port := range createdSlice.Ports {
++					if port.Name != nil {
++						foundPortNames = append(foundPortNames, *port.Name)
++					}
++				}
++
++				if len(foundPortNames) != len(expectedPortNames) {
++					return false, err
++				}
++
++				portSet := sets.NewString(foundPortNames...)
++				expectedPortSet := sets.NewString(expectedPortNames...)
++				return portSet.Equal(expectedPortSet), err
++			}).Should(BeTrue(), "EndpointSlice should contain all unique ports from the Service without duplicates")
++		})
++
++		g.It("Should not panic when Service changes to have a non-nil selector, causing EndpointSlice deletion with no new slices to create", func() {
++			createAndAssertVMI("worker-0-test", "ip-10-32-5-13", "123.45.67.89")
++			createAndAssertTenantSlice("test-epslice", "tenant-service-name", discoveryv1.AddressTypeIPv4,
++				*createPort("http", 80, v1.ProtocolTCP),
++				[]discoveryv1.Endpoint{*createEndpoint("123.45.67.89", "worker-0-test", true, true, false)})
++			createAndAssertInfraServiceLB("infra-service-no-selector", "tenant-service-name", "test-cluster",
++				v1.ServicePort{
++					Name:       "web",
++					Port:       80,
++					NodePort:   31900,
++					Protocol:   v1.ProtocolTCP,
++					TargetPort: intstr.IntOrString{IntVal: 30390},
++				},
++				v1.ServiceExternalTrafficPolicyLocal,
++			)
++
++			// Wait for the controller to create an EndpointSlice in the infra cluster.
++			var epsList *discoveryv1.EndpointSliceList
++			var err error
++			Eventually(func() (bool, error) {
++				epsList, err = testVals.infraClient.DiscoveryV1().EndpointSlices(infraNamespace).
++					List(context.TODO(), metav1.ListOptions{})
++				if err != nil {
++					return false, err
++				}
++				// Wait exactly 1 slice
++				if len(epsList.Items) == 1 {
++					return true, nil
++				}
++				return false, nil
++			}).Should(BeTrue(), "Controller should create an EndpointSlice in infra cluster for the LB service")
++
++			svcWithSelector, err := testVals.infraClient.CoreV1().Services(infraNamespace).
++				Get(context.TODO(), "infra-service-no-selector", metav1.GetOptions{})
++			Expect(err).To(BeNil())
++
++			// Let's set any selector to run the slice deletion logic
++			svcWithSelector.Spec.Selector = map[string]string{"test": "selector-added"}
++			_, err = testVals.infraClient.CoreV1().Services(infraNamespace).
++				Update(context.TODO(), svcWithSelector, metav1.UpdateOptions{})
++			Expect(err).To(BeNil())
++
++			Eventually(func() (bool, error) {
++				epsList, err = testVals.infraClient.DiscoveryV1().EndpointSlices(infraNamespace).
++					List(context.TODO(), metav1.ListOptions{})
++				if err != nil {
++					return false, err
++				}
++				// We expect that after the update service.EndpointSlice will become 0
++				if len(epsList.Items) == 0 {
++					return true, nil
++				}
++				return false, nil
++			}).Should(BeTrue(), "Existing EndpointSlice should be removed because Service now has a selector")
++		})
++
++		g.It("Should remove EndpointSlices and not recreate them when a previously no-selector Service obtains a selector", func() {
++			testVals.infraClient.Fake.PrependReactor("create", "endpointslices", func(action testing.Action) (bool, runtime.Object, error) {
++				createAction := action.(testing.CreateAction)
++				slice := createAction.GetObject().(*discoveryv1.EndpointSlice)
++				if slice.Name == "" && slice.GenerateName != "" {
++					slice.Name = slice.GenerateName + "-fake001"
++				}
++				return false, slice, nil
++			})
++
++			createAndAssertVMI("worker-0-test", "ip-10-32-5-13", "123.45.67.89")
++
++			createAndAssertTenantSlice("test-epslice", "tenant-service-name", discoveryv1.AddressTypeIPv4,
++				*createPort("http", 80, v1.ProtocolTCP),
++				[]discoveryv1.Endpoint{
++					*createEndpoint("123.45.67.89", "worker-0-test", true, true, false),
++				},
++			)
++
++			noSelectorSvcName := "svc-without-selector"
++			svc := &v1.Service{
++				ObjectMeta: metav1.ObjectMeta{
++					Name:      noSelectorSvcName,
++					Namespace: infraNamespace,
++					Labels: map[string]string{
++						kubevirt.TenantServiceNameLabelKey:      "tenant-service-name",
++						kubevirt.TenantServiceNamespaceLabelKey: tenantNamespace,
++						kubevirt.TenantClusterNameLabelKey:      "test-cluster",
++					},
++				},
++				Spec: v1.ServiceSpec{
++					Ports: []v1.ServicePort{
++						{
++							Name:       "web",
++							Port:       80,
++							NodePort:   31900,
++							Protocol:   v1.ProtocolTCP,
++							TargetPort: intstr.IntOrString{IntVal: 30390},
++						},
++					},
++					Type:                  v1.ServiceTypeLoadBalancer,
++					ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal,
++				},
++			}
++
++			_, err := testVals.infraClient.CoreV1().Services(infraNamespace).Create(context.TODO(), svc, metav1.CreateOptions{})
++			Expect(err).To(BeNil())
++
++			Eventually(func() (bool, error) {
++				epsList, err := testVals.infraClient.DiscoveryV1().EndpointSlices(infraNamespace).
++					List(context.TODO(), metav1.ListOptions{})
++				if err != nil {
++					return false, err
++				}
++				return len(epsList.Items) == 1, nil
++			}).Should(BeTrue(), "Controller should create an EndpointSlice in infra cluster for the no-selector LB service")
++
++			svcWithSelector, err := testVals.infraClient.CoreV1().Services(infraNamespace).Get(
++				context.TODO(), noSelectorSvcName, metav1.GetOptions{})
++			Expect(err).To(BeNil())
++
++			svcWithSelector.Spec.Selector = map[string]string{"app": "test-value"}
++			_, err = testVals.infraClient.CoreV1().Services(infraNamespace).
++				Update(context.TODO(), svcWithSelector, metav1.UpdateOptions{})
++			Expect(err).To(BeNil())
++
++			Eventually(func() (bool, error) {
++				epsList, err := testVals.infraClient.DiscoveryV1().EndpointSlices(infraNamespace).
++					List(context.TODO(), metav1.ListOptions{})
++				if err != nil {
++					return false, err
++				}
++				return len(epsList.Items) == 0, nil
++			}).Should(BeTrue(), "All EndpointSlices should be removed after Service acquires a selector (no new slices created)")
++		})
++
++		g.It("Should ignore Services from a different cluster", func() {
++			// Create a Service with cluster label "other-cluster"
++			svc := createInfraServiceLB("infra-service-conflict", "tenant-service-name", "other-cluster",
++				v1.ServicePort{Name: "web", Port: 80, NodePort: 31900, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{IntVal: 30390}},
++				v1.ServiceExternalTrafficPolicyLocal)
++			_, err := testVals.infraClient.CoreV1().Services(infraNamespace).Create(context.TODO(), svc, metav1.CreateOptions{})
++			Expect(err).To(BeNil())
++
++			// The controller should ignore this Service, so no EndpointSlice should be created.
++			Eventually(func() (bool, error) {
++				epsList, err := testVals.infraClient.DiscoveryV1().EndpointSlices(infraNamespace).List(context.TODO(), metav1.ListOptions{})
++				if err != nil {
++					return false, err
++				}
++				// Expect zero slices since cluster label does not match "test-cluster"
++				return len(epsList.Items) == 0, nil
++			}).Should(BeTrue(), "Services with a different cluster label should be ignored")
++		})
++
+ 	})
+ })