[ci] Use shared runners

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

.github/workflows/pull-requests.yaml

@@ -11,7 +11,7 @@ concurrency:
 jobs:
   build:
     name: Build
-    runs-on: [self-hosted]
+    runs-on: [oracle-4cpu-16gb-x86-64]
     permissions:
       contents: read
       packages: write
@@ -34,20 +34,38 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
           registry: ghcr.io
 
+      - name: Install dependencies
+        run: |
+          set -e
+          sudo mkdir -p -m 755 /etc/apt/keyrings
+          sudo wget -O /etc/apt/keyrings/githubcli-archive-keyring.gpg https://cli.github.com/packages/githubcli-archive-keyring.gpg
+          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          sudo apt update
+          sudo apt install -y jq skopeo gh gawk
+          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+          sudo chmod +x /usr/local/bin/yq
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | sudo bash
+          sudo modprobe loop
+
       - name: Build
         run: make build
 
       - name: Build Talos image
         run: make -C packages/core/installer talos-nocloud
-
+     
-      - name: Upload artifacts
+      - name: Upload installer
         uses: actions/upload-artifact@v4
         with:
-          name: cozystack-artefacts
+          name: cozystack-installer
-          path: |
+          path: _out/assets/cozystack-installer.yaml
-            _out/assets/nocloud-amd64.raw.xz
-            _out/assets/cozystack-installer.yaml
 
+      - name: Upload Talos image
+        uses: actions/upload-artifact@v4
+        with:
+          name: talos-image
+          path: _out/assets/nocloud-amd64.raw.xz
+ 
   test:
     name: Test
     runs-on: [self-hosted]
@@ -58,16 +76,16 @@ jobs:
       !contains(github.event.pull_request.labels.*.name, 'release')
 
     steps:
-      - name: Checkout code
+      - name: Download installer
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          fetch-tags: true
-
-      - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
-          name: cozystack-artefacts
+          name: cozystack-installer
+          path: _out/assets/
+
+      - name: Download Talos image
+        uses: actions/download-artifact@v4
+        with:
+          name: talos-image
           path: _out/assets/
 
       - name: Test

.github/workflows/tags.yaml

@@ -15,7 +15,7 @@ concurrency:
 jobs:
   prepare-release:
     name: Prepare Release
-    runs-on: [self-hosted]
+    runs-on: [oracle-4cpu-16gb-x86-64]
     permissions:
       contents: write
       packages: write

hack/e2e-cluster.bats

@@ -86,7 +86,7 @@ EOF
 @test "Prepare VM disks" {
   for i in 1 2 3; do
     cp nocloud-amd64.raw srv${i}/system.img
-    qemu-img resize srv${i}/system.img 20G
+    qemu-img resize srv${i}/system.img 50G
     qemu-img create srv${i}/data.img 100G
   done
 }

packages/apps/clickhouse/Chart.yaml

@@ -23,3 +23,8 @@ version: 0.9.0
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
 appVersion: "24.9.2"
+
+dependencies:
+  - name: cozy-lib
+    version: 0.1.0
+    repository: "http://cozystack.cozy-system.svc/repos/library"

packages/apps/clickhouse/templates/clickhouse.yaml

@@ -122,9 +122,9 @@ spec:
             - name: clickhouse
               image: clickhouse/clickhouse-server:24.9.2.42
               {{- if .Values.resources }}
-              resources: {{- include "cozy-lib.resources.sanitize" .Values.resources | nindent 16 }}
+              resources: {{- include "cozy-lib.resources.sanitize" (list .Values.resources $) | nindent 16 }}
               {{- else if ne .Values.resourcesPreset "none" }}
-              resources: {{- include "cozy-lib.resources.preset" .Values.resourcesPreset | nindent 16 }}
+              resources: {{- include "cozy-lib.resources.preset" (list .Values.resourcesPreset $) | nindent 16 }}
               {{- end }}
               volumeMounts:
                 - name: data-volume-template

packages/library/cozy-lib/Chart.yaml

@@ -15,4 +15,4 @@ type: library
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0

packages/library/cozy-lib/templates/_checkinput.tpl

@@ -0,0 +1,5 @@
+{{- define "cozy-lib.checkInput" }}
+{{-   if not (kindIs "slice" .) }}
+{{-     fail (printf "called cozy-lib function without global scope, expected [<arg>, $], got %s" (kindOf .)) }}
+{{-   end }}
+{{- end }}

packages/library/cozy-lib/templates/_cozyconfig.tpl

@@ -0,0 +1,7 @@
+{{- define "cozy-lib.loadCozyConfig" }}
+{{-   include "cozy-lib.checkInput" . }}
+{{-   if not (hasKey (index . 1) "cozyConfig") }}
+{{-     $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
+{{-     $_ := set (index . 1) "cozyConfig" $cozyConfig }}
+{{-   end }}
+{{- end }}

packages/library/cozy-lib/templates/_resourcepresets.tpl

@@ -11,38 +11,68 @@ These presets are for basic testing and not meant to be used in production
 {{ include "cozy-lib.resources.preset" "nano" -}}
 */}}
 {{- define "cozy-lib.resources.preset" -}}
+{{-   $cpuAllocationRatio := include "cozy-lib.resources.cpuAllocationRatio" . | float64 }}
+{{-   $args := index . 0 }}
+
+{{-   $baseCPU := dict
+        "nano"    (dict "requests" (dict "cpu" "100m" ))
+        "micro"   (dict "requests" (dict "cpu" "250m" ))
+        "small"   (dict "requests" (dict "cpu" "500m" ))
+        "medium"  (dict "requests" (dict "cpu" "500m" ))
+        "large"   (dict "requests" (dict "cpu" "1"    ))
+        "xlarge"  (dict "requests" (dict "cpu" "2"    ))
+        "2xlarge" (dict "requests" (dict "cpu" "4"    ))
+}}
+{{-   $baseMemory := dict
+        "nano"    (dict "requests" (dict "memory" "128Mi" ))
+        "micro"   (dict "requests" (dict "memory" "256Mi" ))
+        "small"   (dict "requests" (dict "memory" "512Mi" ))
+        "medium"  (dict "requests" (dict "memory" "1Gi"   ))
+        "large"   (dict "requests" (dict "memory" "2Gi"   ))
+        "xlarge"  (dict "requests" (dict "memory" "4Gi"   ))
+        "2xlarge" (dict "requests" (dict "memory" "8Gi"   ))
+}}
+
+{{-   range $baseCPU }}
+{{-     $_ := set . "limits" (dict "cpu" (include "cozy-lib.resources.toFloat" .requests.cpu | float64 | mulf $cpuAllocationRatio | toString)) }}
+{{-   end }}
+{{-   range $baseMemory }}
+{{-     $_ := set . "limits" (dict "memory" .requests.memory) }}
+{{-   end }}
+
 {{- $presets := dict 
   "nano" (dict 
-      "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi")
+      "requests" (dict "ephemeral-storage" "50Mi")
-      "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi")
+      "limits" (dict "ephemeral-storage" "2Gi")
    )
   "micro" (dict 
-      "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi")
+      "requests" (dict "ephemeral-storage" "50Mi")
-      "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi")
+      "limits" (dict "ephemeral-storage" "2Gi")
    )
   "small" (dict 
-      "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi")
+      "requests" (dict "ephemeral-storage" "50Mi")
-      "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi")
+      "limits" (dict "ephemeral-storage" "2Gi")
    )
   "medium" (dict 
-      "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi")
+      "requests" (dict "ephemeral-storage" "50Mi")
-      "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi")
+      "limits" (dict "ephemeral-storage" "2Gi")
    )
   "large" (dict 
-      "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi")
+      "requests" (dict "ephemeral-storage" "50Mi")
-      "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi")
+      "limits" (dict "ephemeral-storage" "2Gi")
    )
   "xlarge" (dict 
-      "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi")
+      "requests" (dict "ephemeral-storage" "50Mi")
-      "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi")
+      "limits" (dict "ephemeral-storage" "2Gi")
    )
   "2xlarge" (dict 
-      "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi")
+      "requests" (dict "ephemeral-storage" "50Mi")
-      "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi")
+      "limits" (dict "ephemeral-storage" "2Gi")
    )
  }}
-{{- if hasKey $presets . -}}
+{{- $_ := merge $presets $baseCPU $baseMemory }}
-{{- index $presets . | toYaml -}}
+{{- if hasKey $presets $args -}}
+{{- index $presets $args | toYaml -}}
 {{- else -}}
 {{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" . (join "," (keys $presets)) | fail -}}
 {{- end -}}

packages/library/cozy-lib/templates/_resources.tpl

@@ -1,16 +1,47 @@
+{{- define "cozy-lib.resources.defaultCpuAllocationRatio" }}
+{{-   `10` }}
+{{- end }}
+
+{{- define "cozy-lib.resources.cpuAllocationRatio" }}
+{{-   include "cozy-lib.loadCozyConfig" . }}
+{{-   $cozyConfig := index . 1 "cozyConfig" }}
+{{-   if not $cozyConfig }}
+{{-     include "cozy-lib.resources.defaultCpuAllocationRatio" . }}
+{{-   else }}
+{{-     dig "data" "cpu-allocation-ratio" (include "cozy-lib.resources.defaultCpuAllocationRatio" dict) $cozyConfig }}
+{{-   end }}
+{{- end }}
+
+{{- define "cozy-lib.resources.toFloat" -}}
+    {{- $value := . -}}
+    {{- $unit := 1.0 -}}
+    {{- if typeIs "string" . -}}
+        {{- $base2 := dict "Ki" 0x1p10 "Mi" 0x1p20 "Gi" 0x1p30 "Ti" 0x1p40 "Pi" 0x1p50 "Ei" 0x1p60 -}}
+        {{- $base10 := dict "m" 1e-3 "k" 1e3 "M" 1e6 "G" 1e9 "T" 1e12 "P" 1e15 "E" 1e18 -}}
+        {{- range $k, $v := merge $base2 $base10 -}}
+            {{- if hasSuffix $k $ -}}
+                {{- $value = trimSuffix $k $ -}}
+                {{- $unit = $v -}}
+            {{- end -}}
+        {{- end -}}
+    {{- end -}}
+    {{- mulf (float64 $value) $unit | toString -}}
+{{- end -}}
+
 {{- /*
   A sanitized resource map is a dict with resource-name => resource-quantity.
   If not in such a form, requests are used, then limits. All resources are set
-  to have equal requests and limits, except CPU, that has only requests. The
+  to have equal requests and limits, except CPU, where the limit is increased
-  template expects to receive a dict {"requests":{...}, "limits":{...}} as
+  by a factor of the CPU allocation ratio. The template expects to receive a
-  input, e.g. {{ include "cozy-lib.resources.sanitize" .Values.resources }}.
+  dict {"requests":{...}, "limits":{...}} as input, e.g.
+  {{ include "cozy-lib.resources.sanitize" .Values.resources }}.
   Example input:
   ==============
   limits:
-    cpu: 100m
+    cpu: "1"
     memory: 1024Mi
   requests:
-    cpu: 200m
+    cpu: "2"
     memory: 512Mi
   memory: 256Mi
   devices.com/nvidia: "1"
@@ -18,34 +49,42 @@
   Example output:
   ===============
   limits:
-    devices.com/nvidia: "1"
+    devices.com/nvidia: "1" # only present in top level key
-    memory: 256Mi
+    memory: 256Mi # value from top level key has priority over all others
+    cpu: "2" # value from .requests.cpu has priority over .limits.cpu
   requests:
-    cpu: 200m
+    cpu: 200m # .limits.cpu divided by CPU allocation ratio
-    devices.com/nvidia: "1"
+    devices.com/nvidia: "1" # .requests == .limits
-    memory: 256Mi
+    memory: 256Mi # .requests == .limits
 */}}
 {{- define "cozy-lib.resources.sanitize" }}
+{{-   $cpuAllocationRatio := include "cozy-lib.resources.cpuAllocationRatio" . | float64 }}
 {{-   $sanitizedMap := dict }}
-{{-   if hasKey . "limits" }}
+{{-   $args := index . 0 }}
-{{-     range $k, $v := .limits }}
+{{-   if hasKey $args "limits" }}
+{{-     range $k, $v := $args.limits }}
 {{-       $_ := set $sanitizedMap $k $v }}
 {{-     end }}
 {{-   end }}
-{{-   if hasKey . "requests" }}
+{{-   if hasKey $args "requests" }}
-{{-     range $k, $v := .requests }}
+{{-     range $k, $v := $args.requests }}
 {{-       $_ := set $sanitizedMap $k $v }}
 {{-     end }}
 {{-   end }}
-{{-   range $k, $v := . }}
+{{-   range $k, $v := $args }}
 {{-     if not (or (eq $k "requests") (eq $k "limits")) }}
 {{-       $_ := set $sanitizedMap $k $v }}
 {{-     end }}
 {{-   end }}
 {{-   $output := dict "requests" dict "limits" dict }}
 {{-   range $k, $v := $sanitizedMap }}
-{{-     $_ := set $output.requests $k $v }}
 {{-     if not (eq $k "cpu") }}
+{{-       $_ := set $output.requests $k $v }}
+{{-       $_ := set $output.limits $k $v }}
+{{-     else }}
+{{-       $vcpuRequestF64 := (include "cozy-lib.resources.toFloat" $v) | float64 }}
+{{-       $cpuRequestF64 := divf $vcpuRequestF64 $cpuAllocationRatio }}
+{{-       $_ := set $output.requests $k ($cpuRequestF64 | toString) }}
 {{-       $_ := set $output.limits $k $v }}
 {{-     end }}
 {{-   end }}