From 1b43a5f160637e26023af34ef332ac2e05194ba8 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Mon, 12 May 2025 18:52:32 +0300 Subject: [PATCH 1/2] Remove user-facing config of limits and requests This patch introduces reusable library charts that provide backward-compatibility for users that specify their resources as explicit requests and limits for cpu, however this input is processed so that limits are set equal to requests except for CPU which only gets requests. Users can now embrace the new form by directly specifying resources in the first level of nesting (e.g. resources.cpu=100m instead of .resources.requests.cpu=100m). The order of precedence is top-level, then requests, then limits, ensuring that nothing will break in terms of scheduling, however workloads that specified limits much higher than requests might get a performance hit, now that they cannot use all this excess capacity. This should only affect memory-hungry workloads in low-contention environments. Signed-off-by: Timofei Larkin --- packages/apps/clickhouse/Chart.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/apps/clickhouse/Chart.yaml b/packages/apps/clickhouse/Chart.yaml index bfb34b34..38aabea0 100644 --- a/packages/apps/clickhouse/Chart.yaml +++ b/packages/apps/clickhouse/Chart.yaml @@ -23,3 +23,8 @@ version: 0.9.0 # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. appVersion: "24.9.2" + +dependencies: + - name: cozy-lib + version: 0.1.0 + repository: "http://cozystack.cozy-system.svc/repos/library" From 7023abdba735039f44a66f85babcbe581d897983 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Thu, 22 May 2025 09:33:23 +0300 Subject: [PATCH 2/2] Let users specify CPU requests in VCPUs With this change a request for a virtual machine with 3 vCPUs will reserve exactly the same amount of physical compute, as a request for a Clickhouse instance with `{"resources": {"cpu": "3"}}` in its values, with the scaling factor being KubeVirt's CPU allocation ratio. Signed-off-by: Timofei Larkin --- .../apps/clickhouse/templates/clickhouse.yaml | 4 +- packages/library/cozy-lib/Chart.yaml | 2 +- .../cozy-lib/templates/_checkinput.tpl | 5 ++ .../cozy-lib/templates/_cozyconfig.tpl | 7 ++ .../cozy-lib/templates/_resourcepresets.tpl | 62 +++++++++++----- .../library/cozy-lib/templates/_resources.tpl | 71 ++++++++++++++----- 6 files changed, 116 insertions(+), 35 deletions(-) create mode 100644 packages/library/cozy-lib/templates/_checkinput.tpl create mode 100644 packages/library/cozy-lib/templates/_cozyconfig.tpl diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index dd97188d..dc19ebe4 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -122,9 +122,9 @@ spec: - name: clickhouse image: clickhouse/clickhouse-server:24.9.2.42 {{- if .Values.resources }} - resources: {{- include "cozy-lib.resources.sanitize" .Values.resources | nindent 16 }} + resources: {{- include "cozy-lib.resources.sanitize" (list .Values.resources $) | nindent 16 }} {{- else if ne .Values.resourcesPreset "none" }} - resources: {{- include "cozy-lib.resources.preset" .Values.resourcesPreset | nindent 16 }} + resources: {{- include "cozy-lib.resources.preset" (list .Values.resourcesPreset $) | nindent 16 }} {{- end }} volumeMounts: - name: data-volume-template diff --git a/packages/library/cozy-lib/Chart.yaml b/packages/library/cozy-lib/Chart.yaml index 81a3634d..f9be0259 100644 --- a/packages/library/cozy-lib/Chart.yaml +++ b/packages/library/cozy-lib/Chart.yaml @@ -15,4 +15,4 @@ type: library # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.2.0 diff --git a/packages/library/cozy-lib/templates/_checkinput.tpl b/packages/library/cozy-lib/templates/_checkinput.tpl new file mode 100644 index 00000000..d67bea61 --- /dev/null +++ b/packages/library/cozy-lib/templates/_checkinput.tpl @@ -0,0 +1,5 @@ +{{- define "cozy-lib.checkInput" }} +{{- if not (kindIs "slice" .) }} +{{- fail (printf "called cozy-lib function without global scope, expected [, $], got %s" (kindOf .)) }} +{{- end }} +{{- end }} diff --git a/packages/library/cozy-lib/templates/_cozyconfig.tpl b/packages/library/cozy-lib/templates/_cozyconfig.tpl new file mode 100644 index 00000000..559f7415 --- /dev/null +++ b/packages/library/cozy-lib/templates/_cozyconfig.tpl @@ -0,0 +1,7 @@ +{{- define "cozy-lib.loadCozyConfig" }} +{{- include "cozy-lib.checkInput" . }} +{{- if not (hasKey (index . 1) "cozyConfig") }} +{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} +{{- $_ := set (index . 1) "cozyConfig" $cozyConfig }} +{{- end }} +{{- end }} diff --git a/packages/library/cozy-lib/templates/_resourcepresets.tpl b/packages/library/cozy-lib/templates/_resourcepresets.tpl index f6ecef3f..d26094f0 100644 --- a/packages/library/cozy-lib/templates/_resourcepresets.tpl +++ b/packages/library/cozy-lib/templates/_resourcepresets.tpl @@ -11,38 +11,68 @@ These presets are for basic testing and not meant to be used in production {{ include "cozy-lib.resources.preset" "nano" -}} */}} {{- define "cozy-lib.resources.preset" -}} +{{- $cpuAllocationRatio := include "cozy-lib.resources.cpuAllocationRatio" . | float64 }} +{{- $args := index . 0 }} + +{{- $baseCPU := dict + "nano" (dict "requests" (dict "cpu" "100m" )) + "micro" (dict "requests" (dict "cpu" "250m" )) + "small" (dict "requests" (dict "cpu" "500m" )) + "medium" (dict "requests" (dict "cpu" "500m" )) + "large" (dict "requests" (dict "cpu" "1" )) + "xlarge" (dict "requests" (dict "cpu" "2" )) + "2xlarge" (dict "requests" (dict "cpu" "4" )) +}} +{{- $baseMemory := dict + "nano" (dict "requests" (dict "memory" "128Mi" )) + "micro" (dict "requests" (dict "memory" "256Mi" )) + "small" (dict "requests" (dict "memory" "512Mi" )) + "medium" (dict "requests" (dict "memory" "1Gi" )) + "large" (dict "requests" (dict "memory" "2Gi" )) + "xlarge" (dict "requests" (dict "memory" "4Gi" )) + "2xlarge" (dict "requests" (dict "memory" "8Gi" )) +}} + +{{- range $baseCPU }} +{{- $_ := set . "limits" (dict "cpu" (include "cozy-lib.resources.toFloat" .requests.cpu | float64 | mulf $cpuAllocationRatio | toString)) }} +{{- end }} +{{- range $baseMemory }} +{{- $_ := set . "limits" (dict "memory" .requests.memory) }} +{{- end }} + {{- $presets := dict "nano" (dict - "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi") + "requests" (dict "ephemeral-storage" "50Mi") + "limits" (dict "ephemeral-storage" "2Gi") ) "micro" (dict - "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi") + "requests" (dict "ephemeral-storage" "50Mi") + "limits" (dict "ephemeral-storage" "2Gi") ) "small" (dict - "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi") + "requests" (dict "ephemeral-storage" "50Mi") + "limits" (dict "ephemeral-storage" "2Gi") ) "medium" (dict - "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi") + "requests" (dict "ephemeral-storage" "50Mi") + "limits" (dict "ephemeral-storage" "2Gi") ) "large" (dict - "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi") + "requests" (dict "ephemeral-storage" "50Mi") + "limits" (dict "ephemeral-storage" "2Gi") ) "xlarge" (dict - "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi") + "requests" (dict "ephemeral-storage" "50Mi") + "limits" (dict "ephemeral-storage" "2Gi") ) "2xlarge" (dict - "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi") + "requests" (dict "ephemeral-storage" "50Mi") + "limits" (dict "ephemeral-storage" "2Gi") ) }} -{{- if hasKey $presets . -}} -{{- index $presets . | toYaml -}} +{{- $_ := merge $presets $baseCPU $baseMemory }} +{{- if hasKey $presets $args -}} +{{- index $presets $args | toYaml -}} {{- else -}} {{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" . (join "," (keys $presets)) | fail -}} {{- end -}} diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl index 34ed9ca2..ac848f48 100644 --- a/packages/library/cozy-lib/templates/_resources.tpl +++ b/packages/library/cozy-lib/templates/_resources.tpl @@ -1,16 +1,47 @@ +{{- define "cozy-lib.resources.defaultCpuAllocationRatio" }} +{{- `10` }} +{{- end }} + +{{- define "cozy-lib.resources.cpuAllocationRatio" }} +{{- include "cozy-lib.loadCozyConfig" . }} +{{- $cozyConfig := index . 1 "cozyConfig" }} +{{- if not $cozyConfig }} +{{- include "cozy-lib.resources.defaultCpuAllocationRatio" . }} +{{- else }} +{{- dig "data" "cpu-allocation-ratio" (include "cozy-lib.resources.defaultCpuAllocationRatio" dict) $cozyConfig }} +{{- end }} +{{- end }} + +{{- define "cozy-lib.resources.toFloat" -}} + {{- $value := . -}} + {{- $unit := 1.0 -}} + {{- if typeIs "string" . -}} + {{- $base2 := dict "Ki" 0x1p10 "Mi" 0x1p20 "Gi" 0x1p30 "Ti" 0x1p40 "Pi" 0x1p50 "Ei" 0x1p60 -}} + {{- $base10 := dict "m" 1e-3 "k" 1e3 "M" 1e6 "G" 1e9 "T" 1e12 "P" 1e15 "E" 1e18 -}} + {{- range $k, $v := merge $base2 $base10 -}} + {{- if hasSuffix $k $ -}} + {{- $value = trimSuffix $k $ -}} + {{- $unit = $v -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- mulf (float64 $value) $unit | toString -}} +{{- end -}} + {{- /* A sanitized resource map is a dict with resource-name => resource-quantity. If not in such a form, requests are used, then limits. All resources are set - to have equal requests and limits, except CPU, that has only requests. The - template expects to receive a dict {"requests":{...}, "limits":{...}} as - input, e.g. {{ include "cozy-lib.resources.sanitize" .Values.resources }}. + to have equal requests and limits, except CPU, where the limit is increased + by a factor of the CPU allocation ratio. The template expects to receive a + dict {"requests":{...}, "limits":{...}} as input, e.g. + {{ include "cozy-lib.resources.sanitize" .Values.resources }}. Example input: ============== limits: - cpu: 100m + cpu: "1" memory: 1024Mi requests: - cpu: 200m + cpu: "2" memory: 512Mi memory: 256Mi devices.com/nvidia: "1" @@ -18,34 +49,42 @@ Example output: =============== limits: - devices.com/nvidia: "1" - memory: 256Mi + devices.com/nvidia: "1" # only present in top level key + memory: 256Mi # value from top level key has priority over all others + cpu: "2" # value from .requests.cpu has priority over .limits.cpu requests: - cpu: 200m - devices.com/nvidia: "1" - memory: 256Mi + cpu: 200m # .limits.cpu divided by CPU allocation ratio + devices.com/nvidia: "1" # .requests == .limits + memory: 256Mi # .requests == .limits */}} {{- define "cozy-lib.resources.sanitize" }} +{{- $cpuAllocationRatio := include "cozy-lib.resources.cpuAllocationRatio" . | float64 }} {{- $sanitizedMap := dict }} -{{- if hasKey . "limits" }} -{{- range $k, $v := .limits }} +{{- $args := index . 0 }} +{{- if hasKey $args "limits" }} +{{- range $k, $v := $args.limits }} {{- $_ := set $sanitizedMap $k $v }} {{- end }} {{- end }} -{{- if hasKey . "requests" }} -{{- range $k, $v := .requests }} +{{- if hasKey $args "requests" }} +{{- range $k, $v := $args.requests }} {{- $_ := set $sanitizedMap $k $v }} {{- end }} {{- end }} -{{- range $k, $v := . }} +{{- range $k, $v := $args }} {{- if not (or (eq $k "requests") (eq $k "limits")) }} {{- $_ := set $sanitizedMap $k $v }} {{- end }} {{- end }} {{- $output := dict "requests" dict "limits" dict }} {{- range $k, $v := $sanitizedMap }} -{{- $_ := set $output.requests $k $v }} {{- if not (eq $k "cpu") }} +{{- $_ := set $output.requests $k $v }} +{{- $_ := set $output.limits $k $v }} +{{- else }} +{{- $vcpuRequestF64 := (include "cozy-lib.resources.toFloat" $v) | float64 }} +{{- $cpuRequestF64 := divf $vcpuRequestF64 $cpuAllocationRatio }} +{{- $_ := set $output.requests $k ($cpuRequestF64 | toString) }} {{- $_ := set $output.limits $k $v }} {{- end }} {{- end }}