From 9592f7fe463ff5cfa03b55ac4915bd6fb254cfcb Mon Sep 17 00:00:00 2001
From: Timofei Larkin <lllamnyp@gmail.com>
Date: Mon, 12 May 2025 18:52:32 +0300
Subject: [PATCH] Remove user-facing config of limits and requests

This patch introduces reusable library charts that provide
backward-compatibility for users that specify their resources as
explicit requests and limits for cpu, however this input is processed so
that limits are set equal to requests except for CPU which only gets
requests. Users can now embrace the new form by directly specifying
resources in the first level of nesting (e.g. resources.cpu=100m instead
of .resources.requests.cpu=100m). The order of precedence is top-level,
then requests, then limits, ensuring that nothing will break in terms of
scheduling, however workloads that specified limits much higher than
requests might get a performance hit, now that they cannot use all this
excess capacity. This should only affect memory-hungry workloads in
low-contention environments.

Signed-off-by: Timofei Larkin <lllamnyp@gmail.com>
---
 Makefile                                      |  2 +
 hack/gen_versions_map.sh                      |  8 ++-
 hack/package_chart.sh                         | 65 +++++++++++++++++++
 packages/apps/Makefile                        | 12 +---
 packages/apps/clickhouse/Chart.yaml           |  2 +-
 packages/apps/clickhouse/charts/cozy-lib      |  1 +
 .../apps/clickhouse/templates/clickhouse.yaml |  4 +-
 packages/apps/versions_map                    |  2 +-
 packages/extra/Makefile                       | 12 +---
 packages/library/Makefile                     | 15 +++++
 packages/library/cozy-lib/.helmignore         | 23 +++++++
 packages/library/cozy-lib/Chart.yaml          | 18 +++++
 packages/library/cozy-lib/Makefile            |  6 ++
 packages/library/cozy-lib/README.md           |  1 +
 .../cozy-lib/templates/_resourcepresets.tpl   | 49 ++++++++++++++
 .../library/cozy-lib/templates/_resources.tpl | 53 +++++++++++++++
 packages/library/cozy-lib/values.schema.json  |  5 ++
 packages/library/cozy-lib/values.yaml         |  1 +
 packages/library/testfile                     |  0
 packages/library/versions_map                 |  1 +
 20 files changed, 255 insertions(+), 25 deletions(-)
 create mode 100755 hack/package_chart.sh
 create mode 120000 packages/apps/clickhouse/charts/cozy-lib
 create mode 100644 packages/library/Makefile
 create mode 100644 packages/library/cozy-lib/.helmignore
 create mode 100644 packages/library/cozy-lib/Chart.yaml
 create mode 100644 packages/library/cozy-lib/Makefile
 create mode 100644 packages/library/cozy-lib/README.md
 create mode 100644 packages/library/cozy-lib/templates/_resourcepresets.tpl
 create mode 100644 packages/library/cozy-lib/templates/_resources.tpl
 create mode 100644 packages/library/cozy-lib/values.schema.json
 create mode 100644 packages/library/cozy-lib/values.yaml
 create mode 100644 packages/library/testfile
 create mode 100644 packages/library/versions_map

diff --git a/Makefile b/Makefile
index 99309f4e..56609653 100644
--- a/Makefile
+++ b/Makefile
@@ -28,8 +28,10 @@ build: build-deps
 
 repos:
 	rm -rf _out
+	make -C packages/library check-version-map
 	make -C packages/apps check-version-map
 	make -C packages/extra check-version-map
+	make -C packages/library repo
 	make -C packages/system repo
 	make -C packages/apps repo
 	make -C packages/extra repo
diff --git a/hack/gen_versions_map.sh b/hack/gen_versions_map.sh
index 5169bc2f..60ede8ba 100755
--- a/hack/gen_versions_map.sh
+++ b/hack/gen_versions_map.sh
@@ -16,13 +16,15 @@ if [ ! -f "$file" ] || [ ! -s "$file" ]; then
   exit 0
 fi
 
-miss_map=$(echo "$new_map" | awk 'NR==FNR { nm[$1 " " $2] = $3; next } { if (!($1 " " $2 in nm)) print $1, $2, $3}' - "$file")
+miss_map=$(mktemp)
+trap 'rm -f "$miss_map"' EXIT
+echo -n "$new_map" | awk 'NR==FNR { nm[$1 " " $2] = $3; next } { if (!($1 " " $2 in nm)) print $1, $2, $3}' - "$file" > $miss_map
 
 # search accross all tags sorted by version
 search_commits=$(git ls-remote --tags origin | awk -F/ '$3 ~ /v[0-9]+.[0-9]+.[0-9]+/ {print}' | sort -k2,2 -rV | awk '{print $1}')
 
 resolved_miss_map=$(
-  echo "$miss_map" | while read -r chart version commit; do
+  while read -r chart version commit; do
     # if version is found in HEAD, it's HEAD
     if [ "$(awk '$1 == "version:" {print $2}' ./${chart}/Chart.yaml)" = "${version}" ]; then
       echo "$chart $version HEAD"
@@ -56,7 +58,7 @@ resolved_miss_map=$(
     fi
     
     echo "$chart $version $found_tag"
-  done
+  done < $miss_map
 )
 
 printf "%s\n" "$new_map" "$resolved_miss_map" | sort -k1,1 -k2,2 -V | awk '$1' > "$file"
diff --git a/hack/package_chart.sh b/hack/package_chart.sh
new file mode 100755
index 00000000..f8ab297f
--- /dev/null
+++ b/hack/package_chart.sh
@@ -0,0 +1,65 @@
+#!/bin/sh
+
+set -e
+
+usage() {
+        printf "%s\n" "Usage:" >&2 ;
+        printf -- "%s\n" '---' >&2 ;
+        printf "%s %s\n" "$0" "INPUT_DIR OUTPUT_DIR TMP_DIR [DEPENDENCY_DIR]" >&2 ;
+        printf -- "%s\n" '---' >&2 ;
+        printf "%s\n" "Takes a helm repository from INPUT_DIR, with an optional library repository in" >&2 ;
+        printf "%s\n" "DEPENDENCY_DIR, prepares a view of the git archive at select points in history" >&2 ;
+        printf "%s\n" "in TMP_DIR and packages helm charts, outputting the tarballs to OUTPUT_DIR" >&2 ;
+}
+
+if [ "x$(basename $PWD)" != "xpackages" ]
+then
+        echo "Error: This script must run from the ./packages/ directory" >&2
+        echo >&2
+        usage
+        exit 1
+fi
+
+if [ "x$#" != "x3" ] && [ "x$#" != "x4" ]
+then
+        echo "Error: This script takes 3 or 4 arguments" >&2
+        echo "Got $# arguments:" "$@" >&2
+        echo >&2
+        usage
+        exit 1
+fi
+
+input_dir=$1
+output_dir=$2
+tmp_dir=$3
+
+if [ "x$#" = "x4" ]
+then
+        dependency_dir=$4
+fi
+
+rm -rf "${output_dir:?}"
+mkdir -p "${output_dir}"
+while read package _ commit
+do
+        # this lets devs build the packages from a dirty repo for quick local testing
+        if [ "x$commit" = "xHEAD" ]
+        then
+                helm package "${input_dir}/${package}" -d "${output_dir}"
+                continue
+        fi
+        git archive --format tar "${commit}" "${input_dir}/${package}" | tar -xf- -C "${tmp_dir}/"
+
+        # the library chart is not present in older commits and git archive doesn't fail gracefully if the path is not found
+        if [ "x${dependency_dir}" != "x" ] && git ls-tree --name-only "${commit}" "${dependency_dir}" | grep -qx "${dependency_dir}"
+        then
+                git archive --format tar "${commit}" "${dependency_dir}" | tar -xf- -C "${tmp_dir}/"
+        fi
+        helm package "${tmp_dir}/${input_dir}/${package}" -d "${output_dir}"
+        rm -rf "${tmp_dir:?}/${input_dir:?}/${package:?}"
+        if [ "x${dependency_dir}" != "x" ]
+        then
+                rm -rf "${tmp_dir:?}/${dependency_dir:?}"
+        fi
+done < "${input_dir}/versions_map"
+helm repo index "${output_dir}"
diff --git a/packages/apps/Makefile b/packages/apps/Makefile
index 506c8750..8751a048 100644
--- a/packages/apps/Makefile
+++ b/packages/apps/Makefile
@@ -1,14 +1,8 @@
-OUT=../../_out/repos/apps
-TMP=../../_out/repos/apps/historical
+OUT=../_out/repos/apps
+TMP := $(shell mktemp -d)
 
 repo:
-	rm -rf "$(OUT)"
-	mkdir -p "$(OUT)"
-	awk '$$3 != "HEAD" {print "mkdir -p $(TMP)/" $$1 "-" $$2}' versions_map | sh -ex
-	awk '$$3 != "HEAD" {print "git archive " $$3 " " $$1 " | tar -xf- --strip-components=1 -C $(TMP)/" $$1 "-" $$2 }' versions_map | sh -ex
-	helm package -d "$(OUT)" $$(find . $(TMP) -mindepth 2 -maxdepth 2 -name Chart.yaml | awk 'sub("/Chart.yaml", "")' | sort -V)
-	cd "$(OUT)" && helm repo index . --url http://cozystack.cozy-system.svc/repos/apps
-	rm -rf "$(TMP)"
+	cd .. && ../hack/package_chart.sh apps $(OUT) $(TMP) library
 
 fix-chartnames:
 	find . -maxdepth 2 -name Chart.yaml  | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done
diff --git a/packages/apps/clickhouse/Chart.yaml b/packages/apps/clickhouse/Chart.yaml
index 935cf6af..bfb34b34 100644
--- a/packages/apps/clickhouse/Chart.yaml
+++ b/packages/apps/clickhouse/Chart.yaml
@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.8.0
+version: 0.9.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
diff --git a/packages/apps/clickhouse/charts/cozy-lib b/packages/apps/clickhouse/charts/cozy-lib
new file mode 120000
index 00000000..e1813509
--- /dev/null
+++ b/packages/apps/clickhouse/charts/cozy-lib
@@ -0,0 +1 @@
+../../../library/cozy-lib
\ No newline at end of file
diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml
index dd4b0fdb..dd97188d 100644
--- a/packages/apps/clickhouse/templates/clickhouse.yaml
+++ b/packages/apps/clickhouse/templates/clickhouse.yaml
@@ -122,9 +122,9 @@ spec:
             - name: clickhouse
               image: clickhouse/clickhouse-server:24.9.2.42
               {{- if .Values.resources }}
-              resources: {{- toYaml .Values.resources | nindent 16 }}
+              resources: {{- include "cozy-lib.resources.sanitize" .Values.resources | nindent 16 }}
               {{- else if ne .Values.resourcesPreset "none" }}
-              resources: {{- include "resources.preset" (dict "type" .Values.resourcesPreset "Release" .Release) | nindent 16 }}
+              resources: {{- include "cozy-lib.resources.preset" .Values.resourcesPreset | nindent 16 }}
               {{- end }}
               volumeMounts:
                 - name: data-volume-template
diff --git a/packages/apps/versions_map b/packages/apps/versions_map
index 89d9990d..776c897d 100644
--- a/packages/apps/versions_map
+++ b/packages/apps/versions_map
@@ -9,7 +9,7 @@ clickhouse 0.6.0 1ec10165
 clickhouse 0.6.1 c62a83a7
 clickhouse 0.6.2 8267072d
 clickhouse 0.7.0 93bdf411
-clickhouse 0.8.0 HEAD
+clickhouse 0.9.0 HEAD
 ferretdb 0.1.0 e9716091
 ferretdb 0.1.1 91b0499a
 ferretdb 0.2.0 6c5cf5bf
diff --git a/packages/extra/Makefile b/packages/extra/Makefile
index 5586c59e..e05e4985 100644
--- a/packages/extra/Makefile
+++ b/packages/extra/Makefile
@@ -1,14 +1,8 @@
-OUT=../../_out/repos/extra
-TMP=../../_out/repos/extra/historical
+OUT=../_out/repos/extra
+TMP := $(shell mktemp -d)
 
 repo:
-	rm -rf "$(OUT)"
-	mkdir -p "$(OUT)"
-	awk '$$3 != "HEAD" {print "mkdir -p $(TMP)/" $$1 "-" $$2}' versions_map | sh -ex
-	awk '$$3 != "HEAD" {print "git archive " $$3 " " $$1 " | tar -xf- --strip-components=1 -C $(TMP)/" $$1 "-" $$2 }' versions_map | sh -ex
-	helm package -d "$(OUT)" $$(find . $(TMP) -mindepth 2 -maxdepth 2 -name Chart.yaml | awk 'sub("/Chart.yaml", "")' | sort -V)
-	cd "$(OUT)" && helm repo index . --url http://cozystack.cozy-system.svc/repos/extra
-	rm -rf "$(TMP)"
+	cd .. && ../hack/package_chart.sh extra $(OUT) $(TMP) library
 
 fix-chartnames:
 	find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done
diff --git a/packages/library/Makefile b/packages/library/Makefile
new file mode 100644
index 00000000..664771cf
--- /dev/null
+++ b/packages/library/Makefile
@@ -0,0 +1,15 @@
+OUT=../_out/repos/library
+TMP := $(shell mktemp -d)
+
+repo:
+	cd .. && ../hack/package_chart.sh library $(OUT) $(TMP)
+
+fix-chartnames:
+	find . -maxdepth 2 -name Chart.yaml  | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done
+
+gen-versions-map: fix-chartnames
+	../../hack/gen_versions_map.sh
+
+check-version-map: gen-versions-map
+	git diff --exit-code -- versions_map
+
diff --git a/packages/library/cozy-lib/.helmignore b/packages/library/cozy-lib/.helmignore
new file mode 100644
index 00000000..0e8a0eb3
--- /dev/null
+++ b/packages/library/cozy-lib/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/packages/library/cozy-lib/Chart.yaml b/packages/library/cozy-lib/Chart.yaml
new file mode 100644
index 00000000..81a3634d
--- /dev/null
+++ b/packages/library/cozy-lib/Chart.yaml
@@ -0,0 +1,18 @@
+apiVersion: v2
+name: cozy-lib
+description: Common Cozystack templates
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: library
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
diff --git a/packages/library/cozy-lib/Makefile b/packages/library/cozy-lib/Makefile
new file mode 100644
index 00000000..fa0142de
--- /dev/null
+++ b/packages/library/cozy-lib/Makefile
@@ -0,0 +1,6 @@
+include ../../../scripts/common-envs.mk
+include ../../../scripts/package.mk
+
+generate:
+	readme-generator -v values.yaml -s values.schema.json -r README.md
+
diff --git a/packages/library/cozy-lib/README.md b/packages/library/cozy-lib/README.md
new file mode 100644
index 00000000..d4bf18ad
--- /dev/null
+++ b/packages/library/cozy-lib/README.md
@@ -0,0 +1 @@
+## Parameters
diff --git a/packages/library/cozy-lib/templates/_resourcepresets.tpl b/packages/library/cozy-lib/templates/_resourcepresets.tpl
new file mode 100644
index 00000000..f6ecef3f
--- /dev/null
+++ b/packages/library/cozy-lib/templates/_resourcepresets.tpl
@@ -0,0 +1,49 @@
+{{/*
+Copyright Broadcom, Inc. All Rights Reserved.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Return a resource request/limit object based on a given preset.
+These presets are for basic testing and not meant to be used in production
+{{ include "cozy-lib.resources.preset" "nano" -}}
+*/}}
+{{- define "cozy-lib.resources.preset" -}}
+{{- $presets := dict 
+  "nano" (dict 
+      "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi")
+   )
+  "micro" (dict 
+      "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi")
+   )
+  "small" (dict 
+      "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi")
+   )
+  "medium" (dict 
+      "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi")
+   )
+  "large" (dict 
+      "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi")
+   )
+  "xlarge" (dict 
+      "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi")
+   )
+  "2xlarge" (dict 
+      "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi")
+      "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi")
+   )
+ }}
+{{- if hasKey $presets . -}}
+{{- index $presets . | toYaml -}}
+{{- else -}}
+{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" . (join "," (keys $presets)) | fail -}}
+{{- end -}}
+{{- end -}}
diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl
new file mode 100644
index 00000000..34ed9ca2
--- /dev/null
+++ b/packages/library/cozy-lib/templates/_resources.tpl
@@ -0,0 +1,53 @@
+{{- /*
+  A sanitized resource map is a dict with resource-name => resource-quantity.
+  If not in such a form, requests are used, then limits. All resources are set
+  to have equal requests and limits, except CPU, that has only requests. The
+  template expects to receive a dict {"requests":{...}, "limits":{...}} as
+  input, e.g. {{ include "cozy-lib.resources.sanitize" .Values.resources }}.
+  Example input:
+  ==============
+  limits:
+    cpu: 100m
+    memory: 1024Mi
+  requests:
+    cpu: 200m
+    memory: 512Mi
+  memory: 256Mi
+  devices.com/nvidia: "1"
+
+  Example output:
+  ===============
+  limits:
+    devices.com/nvidia: "1"
+    memory: 256Mi
+  requests:
+    cpu: 200m
+    devices.com/nvidia: "1"
+    memory: 256Mi
+*/}}
+{{- define "cozy-lib.resources.sanitize" }}
+{{-   $sanitizedMap := dict }}
+{{-   if hasKey . "limits" }}
+{{-     range $k, $v := .limits }}
+{{-       $_ := set $sanitizedMap $k $v }}
+{{-     end }}
+{{-   end }}
+{{-   if hasKey . "requests" }}
+{{-     range $k, $v := .requests }}
+{{-       $_ := set $sanitizedMap $k $v }}
+{{-     end }}
+{{-   end }}
+{{-   range $k, $v := . }}
+{{-     if not (or (eq $k "requests") (eq $k "limits")) }}
+{{-       $_ := set $sanitizedMap $k $v }}
+{{-     end }}
+{{-   end }}
+{{-   $output := dict "requests" dict "limits" dict }}
+{{-   range $k, $v := $sanitizedMap }}
+{{-     $_ := set $output.requests $k $v }}
+{{-     if not (eq $k "cpu") }}
+{{-       $_ := set $output.limits $k $v }}
+{{-     end }}
+{{-   end }}
+{{-   $output | toYaml }}
+{{- end  }}
diff --git a/packages/library/cozy-lib/values.schema.json b/packages/library/cozy-lib/values.schema.json
new file mode 100644
index 00000000..decc79aa
--- /dev/null
+++ b/packages/library/cozy-lib/values.schema.json
@@ -0,0 +1,5 @@
+{
+    "title": "Chart Values",
+    "type": "object",
+    "properties": {}
+}
\ No newline at end of file
diff --git a/packages/library/cozy-lib/values.yaml b/packages/library/cozy-lib/values.yaml
new file mode 100644
index 00000000..0967ef42
--- /dev/null
+++ b/packages/library/cozy-lib/values.yaml
@@ -0,0 +1 @@
+{}
diff --git a/packages/library/testfile b/packages/library/testfile
new file mode 100644
index 00000000..e69de29b
diff --git a/packages/library/versions_map b/packages/library/versions_map
new file mode 100644
index 00000000..63834fdd
--- /dev/null
+++ b/packages/library/versions_map
@@ -0,0 +1 @@
+cozy-lib 0.1.0 HEAD