Remove user-facing config of limits and requests (#935)

This patch introduces reusable library charts that provide backward-compatibility for users that specify their resources as explicit requests and limits for cpu, however this input is processed so that limits are set equal to requests except for CPU which only gets requests. Users can now embrace the new form by directly specifying resources in the first level of nesting (e.g. resources.cpu=100m instead of .resources.requests.cpu=100m). The order of precedence is top-level, then requests, then limits, ensuring that nothing will break in terms of scheduling, however workloads that specified limits much higher than requests might get a performance hit, now that they cannot use all this excess capacity. This should only affect memory-hungry workloads in low-contention environments.  ## Summary by CodeRabbit - **New Features** - Introduced a reusable Helm library chart, "cozy-lib", providing common templates and resource helpers for other charts. - Added resource preset and sanitization templates to standardize Kubernetes resource configurations. - ClickHouse chart now depends on "cozy-lib" for improved resource handling. - Added a new packaging script and streamlined Helm chart packaging processes across multiple packages. - **Bug Fixes** - Resource configuration logic in the ClickHouse deployment was updated to use the new library templates, ensuring more consistent resource definitions. - **Chores** - Added new Makefiles and version mapping for streamlined Helm chart packaging and validation. - Updated ClickHouse chart version to 0.9.0 and reflected this in version mapping files. - Refactored Makefile targets to consolidate packaging logic and improve maintainability.
2026-01-27 10:18:39 +00:00 · 2025-05-23 18:11:27 +03:00
parent 609e7ede86 9592f7fe46
commit a8ba6b1328
20 changed files with 255 additions and 25 deletions
--- a/2
+++ b/2
@@ -29,8 +29,10 @@ build: build-deps

 repos:
 	rm -rf _out
+	make -C packages/library check-version-map
 	make -C packages/apps check-version-map
 	make -C packages/extra check-version-map
+	make -C packages/library repo
 	make -C packages/system repo
 	make -C packages/apps repo
 	make -C packages/extra repo
--- a/hack/gen_versions_map.sh
+++ b/hack/gen_versions_map.sh
@@ -16,13 +16,15 @@ if [ ! -f "$file" ] || [ ! -s "$file" ]; then
  exit 0
 fi

-miss_map=$(echo "$new_map" | awk 'NR==FNR { nm[$1 " " $2] = $3; next } { if (!($1 " " $2 in nm)) print $1, $2, $3}' - "$file")
+miss_map=$(mktemp)
+trap 'rm -f "$miss_map"' EXIT
+echo -n "$new_map" | awk 'NR==FNR { nm[$1 " " $2] = $3; next } { if (!($1 " " $2 in nm)) print $1, $2, $3}' - "$file" > $miss_map

 # search accross all tags sorted by version
 search_commits=$(git ls-remote --tags origin | awk -F/ '$3 ~ /v[0-9]+.[0-9]+.[0-9]+/ {print}' | sort -k2,2 -rV | awk '{print $1}')

 resolved_miss_map=$(
-  echo "$miss_map" | while read -r chart version commit; do
+  while read -r chart version commit; do
    # if version is found in HEAD, it's HEAD
    if [ "$(awk '$1 == "version:" {print $2}' ./${chart}/Chart.yaml)" = "${version}" ]; then
      echo "$chart $version HEAD"
@@ -56,7 +58,7 @@ resolved_miss_map=$(
    fi
    
    echo "$chart $version $found_tag"
-  done
+  done < $miss_map
 )

 printf "%s\n" "$new_map" "$resolved_miss_map" | sort -k1,1 -k2,2 -V | awk '$1' > "$file"
--- a/hack/package_chart.sh
+++ b/hack/package_chart.sh
@@ -0,0 +1,65 @@
+#!/bin/sh
+
+set -e
+
+usage() {
+        printf "%s\n" "Usage:" >&2 ;
+        printf -- "%s\n" '---' >&2 ;
+        printf "%s %s\n" "$0" "INPUT_DIR OUTPUT_DIR TMP_DIR [DEPENDENCY_DIR]" >&2 ;
+        printf -- "%s\n" '---' >&2 ;
+        printf "%s\n" "Takes a helm repository from INPUT_DIR, with an optional library repository in" >&2 ;
+        printf "%s\n" "DEPENDENCY_DIR, prepares a view of the git archive at select points in history" >&2 ;
+        printf "%s\n" "in TMP_DIR and packages helm charts, outputting the tarballs to OUTPUT_DIR" >&2 ;
+}
+
+if [ "x$(basename $PWD)" != "xpackages" ]
+then
+        echo "Error: This script must run from the ./packages/ directory" >&2
+        echo >&2
+        usage
+        exit 1
+fi
+
+if [ "x$#" != "x3" ] && [ "x$#" != "x4" ]
+then
+        echo "Error: This script takes 3 or 4 arguments" >&2
+        echo "Got $# arguments:" "$@" >&2
+        echo >&2
+        usage
+        exit 1
+fi
+
+input_dir=$1
+output_dir=$2
+tmp_dir=$3
+
+if [ "x$#" = "x4" ]
+then
+        dependency_dir=$4
+fi
+
+rm -rf "${output_dir:?}"
+mkdir -p "${output_dir}"
+while read package _ commit
+do
+        # this lets devs build the packages from a dirty repo for quick local testing
+        if [ "x$commit" = "xHEAD" ]
+        then
+                helm package "${input_dir}/${package}" -d "${output_dir}"
+                continue
+        fi
+        git archive --format tar "${commit}" "${input_dir}/${package}" | tar -xf- -C "${tmp_dir}/"
+
+        # the library chart is not present in older commits and git archive doesn't fail gracefully if the path is not found
+        if [ "x${dependency_dir}" != "x" ] && git ls-tree --name-only "${commit}" "${dependency_dir}" | grep -qx "${dependency_dir}"
+        then
+                git archive --format tar "${commit}" "${dependency_dir}" | tar -xf- -C "${tmp_dir}/"
+        fi
+        helm package "${tmp_dir}/${input_dir}/${package}" -d "${output_dir}"
+        rm -rf "${tmp_dir:?}/${input_dir:?}/${package:?}"
+        if [ "x${dependency_dir}" != "x" ]
+        then
+                rm -rf "${tmp_dir:?}/${dependency_dir:?}"
+        fi
+done < "${input_dir}/versions_map"
+helm repo index "${output_dir}"
--- a/packages/apps/Makefile
+++ b/packages/apps/Makefile
@@ -1,14 +1,8 @@
-OUT=../../_out/repos/apps
-TMP=../../_out/repos/apps/historical
+OUT=../_out/repos/apps
+TMP := $(shell mktemp -d)

 repo:
-	rm -rf "$(OUT)"
-	mkdir -p "$(OUT)"
-	awk '$$3 != "HEAD" {print "mkdir -p $(TMP)/" $$1 "-" $$2}' versions_map | sh -ex
-	awk '$$3 != "HEAD" {print "git archive " $$3 " " $$1 " | tar -xf- --strip-components=1 -C $(TMP)/" $$1 "-" $$2 }' versions_map | sh -ex
-	helm package -d "$(OUT)" $$(find . $(TMP) -mindepth 2 -maxdepth 2 -name Chart.yaml | awk 'sub("/Chart.yaml", "")' | sort -V)
-	cd "$(OUT)" && helm repo index . --url http://cozystack.cozy-system.svc/repos/apps
-	rm -rf "$(TMP)"
+	cd .. && ../hack/package_chart.sh apps $(OUT) $(TMP) library

 fix-chartnames:
 	find . -maxdepth 2 -name Chart.yaml  | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done
--- a/packages/apps/clickhouse/Chart.yaml
+++ b/packages/apps/clickhouse/Chart.yaml
@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.8.0
+version: 0.9.0

 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
--- a/packages/apps/clickhouse/charts/cozy-lib
+++ b/packages/apps/clickhouse/charts/cozy-lib
@@ -0,0 +1 @@
+../../../library/cozy-lib
--- a/packages/apps/clickhouse/templates/clickhouse.yaml
+++ b/packages/apps/clickhouse/templates/clickhouse.yaml
@@ -122,9 +122,9 @@ spec:
            - name: clickhouse
              image: clickhouse/clickhouse-server:24.9.2.42
              {{- if .Values.resources }}
-              resources: {{- toYaml .Values.resources | nindent 16 }}
+              resources: {{- include "cozy-lib.resources.sanitize" .Values.resources | nindent 16 }}
              {{- else if ne .Values.resourcesPreset "none" }}
-              resources: {{- include "resources.preset" (dict "type" .Values.resourcesPreset "Release" .Release) | nindent 16 }}
+              resources: {{- include "cozy-lib.resources.preset" .Values.resourcesPreset | nindent 16 }}
              {{- end }}
              volumeMounts:
                - name: data-volume-template
--- a/packages/apps/versions_map
+++ b/packages/apps/versions_map
@@ -9,7 +9,7 @@ clickhouse 0.6.0 1ec10165
 clickhouse 0.6.1 c62a83a7
 clickhouse 0.6.2 8267072d
 clickhouse 0.7.0 93bdf411
-clickhouse 0.8.0 HEAD
+clickhouse 0.9.0 HEAD
 ferretdb 0.1.0 e9716091
 ferretdb 0.1.1 91b0499a
 ferretdb 0.2.0 6c5cf5bf
--- a/packages/extra/Makefile
+++ b/packages/extra/Makefile
@@ -1,14 +1,8 @@
-OUT=../../_out/repos/extra
-TMP=../../_out/repos/extra/historical
+OUT=../_out/repos/extra
+TMP := $(shell mktemp -d)

 repo:
-	rm -rf "$(OUT)"
-	mkdir -p "$(OUT)"
-	awk '$$3 != "HEAD" {print "mkdir -p $(TMP)/" $$1 "-" $$2}' versions_map | sh -ex
-	awk '$$3 != "HEAD" {print "git archive " $$3 " " $$1 " | tar -xf- --strip-components=1 -C $(TMP)/" $$1 "-" $$2 }' versions_map | sh -ex
-	helm package -d "$(OUT)" $$(find . $(TMP) -mindepth 2 -maxdepth 2 -name Chart.yaml | awk 'sub("/Chart.yaml", "")' | sort -V)
-	cd "$(OUT)" && helm repo index . --url http://cozystack.cozy-system.svc/repos/extra
-	rm -rf "$(TMP)"
+	cd .. && ../hack/package_chart.sh extra $(OUT) $(TMP) library

 fix-chartnames:
 	find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done
--- a/packages/library/Makefile
+++ b/packages/library/Makefile
@@ -0,0 +1,15 @@
+OUT=../_out/repos/library
+TMP := $(shell mktemp -d)
+
+repo:
+	cd .. && ../hack/package_chart.sh library $(OUT) $(TMP)
+
+fix-chartnames:
+	find . -maxdepth 2 -name Chart.yaml  | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done
+
+gen-versions-map: fix-chartnames
+	../../hack/gen_versions_map.sh
+
+check-version-map: gen-versions-map
+	git diff --exit-code -- versions_map
+
--- a/packages/library/cozy-lib/.helmignore
+++ b/packages/library/cozy-lib/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
--- a/packages/library/cozy-lib/Chart.yaml
+++ b/packages/library/cozy-lib/Chart.yaml
@@ -0,0 +1,18 @@
+apiVersion: v2
+name: cozy-lib
+description: Common Cozystack templates
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: library
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
--- a/packages/library/cozy-lib/Makefile
+++ b/packages/library/cozy-lib/Makefile
@@ -0,0 +1,6 @@
+include ../../../scripts/common-envs.mk
+include ../../../scripts/package.mk
+
+generate:
+	readme-generator -v values.yaml -s values.schema.json -r README.md
+
--- a/packages/library/cozy-lib/README.md
+++ b/packages/library/cozy-lib/README.md
@@ -0,0 +1 @@
+## Parameters
--- a/packages/library/cozy-lib/templates/_resourcepresets.tpl
+++ b/packages/library/cozy-lib/templates/_resourcepresets.tpl
@@ -0,0 +1,49 @@
+{{/*
+Copyright Broadcom, Inc. All Rights Reserved.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Return a resource request/limit object based on a given preset.
+These presets are for basic testing and not meant to be used in production
+{{ include "cozy-lib.resources.preset" "nano" -}}
+*/}}
+{{- define "cozy-lib.resources.preset" -}}
+{{- $presets := dict 
+  "nano" (dict 
+      "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi")
+   )
+  "micro" (dict 
+      "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi")
+   )
+  "small" (dict 
+      "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi")
+   )
+  "medium" (dict 
+      "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi")
+   )
+  "large" (dict 
+      "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi")
+   )
+  "xlarge" (dict 
+      "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi")
+   )
+  "2xlarge" (dict 
+      "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi")
+   )
+ }}
+{{- if hasKey $presets . -}}
+{{- index $presets . | toYaml -}}
+{{- else -}}
+{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" . (join "," (keys $presets)) | fail -}}
+{{- end -}}
+{{- end -}}
--- a/packages/library/cozy-lib/templates/_resources.tpl
+++ b/packages/library/cozy-lib/templates/_resources.tpl
@@ -0,0 +1,53 @@
+{{- /*
+  A sanitized resource map is a dict with resource-name => resource-quantity.
+  If not in such a form, requests are used, then limits. All resources are set
+  to have equal requests and limits, except CPU, that has only requests. The
+  template expects to receive a dict {"requests":{...}, "limits":{...}} as
+  input, e.g. {{ include "cozy-lib.resources.sanitize" .Values.resources }}.
+  Example input:
+  ==============
+  limits:
+    cpu: 100m
+    memory: 1024Mi
+  requests:
+    cpu: 200m
+    memory: 512Mi
+  memory: 256Mi
+  devices.com/nvidia: "1"
+
+  Example output:
+  ===============
+  limits:
+    devices.com/nvidia: "1"
+    memory: 256Mi
+  requests:
+    cpu: 200m
+    devices.com/nvidia: "1"
+    memory: 256Mi
+*/}}
+{{- define "cozy-lib.resources.sanitize" }}
+{{-   $sanitizedMap := dict }}
+{{-   if hasKey . "limits" }}
+{{-     range $k, $v := .limits }}
+{{-       $_ := set $sanitizedMap $k $v }}
+{{-     end }}
+{{-   end }}
+{{-   if hasKey . "requests" }}
+{{-     range $k, $v := .requests }}
+{{-       $_ := set $sanitizedMap $k $v }}
+{{-     end }}
+{{-   end }}
+{{-   range $k, $v := . }}
+{{-     if not (or (eq $k "requests") (eq $k "limits")) }}
+{{-       $_ := set $sanitizedMap $k $v }}
+{{-     end }}
+{{-   end }}
+{{-   $output := dict "requests" dict "limits" dict }}
+{{-   range $k, $v := $sanitizedMap }}
+{{-     $_ := set $output.requests $k $v }}
+{{-     if not (eq $k "cpu") }}
+{{-       $_ := set $output.limits $k $v }}
+{{-     end }}
+{{-   end }}
+{{-   $output | toYaml }}
+{{- end  }}
--- a/packages/library/cozy-lib/values.schema.json
+++ b/packages/library/cozy-lib/values.schema.json
@@ -0,0 +1,5 @@
+{
+    "title": "Chart Values",
+    "type": "object",
+    "properties": {}
+}
--- a/packages/library/cozy-lib/values.yaml
+++ b/packages/library/cozy-lib/values.yaml
@@ -0,0 +1 @@
+{}
--- a/packages/library/testfile
+++ b/packages/library/testfile
--- a/packages/library/versions_map
+++ b/packages/library/versions_map
@@ -0,0 +1 @@
+cozy-lib 0.1.0 HEAD