diff --git a/Makefile b/Makefile index 63f61458..c1b4ed72 100644 --- a/Makefile +++ b/Makefile @@ -29,8 +29,10 @@ build: build-deps repos: rm -rf _out + make -C packages/library check-version-map make -C packages/apps check-version-map make -C packages/extra check-version-map + make -C packages/library repo make -C packages/system repo make -C packages/apps repo make -C packages/extra repo diff --git a/hack/gen_versions_map.sh b/hack/gen_versions_map.sh index 5169bc2f..60ede8ba 100755 --- a/hack/gen_versions_map.sh +++ b/hack/gen_versions_map.sh @@ -16,13 +16,15 @@ if [ ! -f "$file" ] || [ ! -s "$file" ]; then exit 0 fi -miss_map=$(echo "$new_map" | awk 'NR==FNR { nm[$1 " " $2] = $3; next } { if (!($1 " " $2 in nm)) print $1, $2, $3}' - "$file") +miss_map=$(mktemp) +trap 'rm -f "$miss_map"' EXIT +echo -n "$new_map" | awk 'NR==FNR { nm[$1 " " $2] = $3; next } { if (!($1 " " $2 in nm)) print $1, $2, $3}' - "$file" > $miss_map # search accross all tags sorted by version search_commits=$(git ls-remote --tags origin | awk -F/ '$3 ~ /v[0-9]+.[0-9]+.[0-9]+/ {print}' | sort -k2,2 -rV | awk '{print $1}') resolved_miss_map=$( - echo "$miss_map" | while read -r chart version commit; do + while read -r chart version commit; do # if version is found in HEAD, it's HEAD if [ "$(awk '$1 == "version:" {print $2}' ./${chart}/Chart.yaml)" = "${version}" ]; then echo "$chart $version HEAD" @@ -56,7 +58,7 @@ resolved_miss_map=$( fi echo "$chart $version $found_tag" - done + done < $miss_map ) printf "%s\n" "$new_map" "$resolved_miss_map" | sort -k1,1 -k2,2 -V | awk '$1' > "$file" diff --git a/hack/package_chart.sh b/hack/package_chart.sh new file mode 100755 index 00000000..f8ab297f --- /dev/null +++ b/hack/package_chart.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +set -e + +usage() { + printf "%s\n" "Usage:" >&2 ; + printf -- "%s\n" '---' >&2 ; + printf "%s %s\n" "$0" "INPUT_DIR OUTPUT_DIR TMP_DIR [DEPENDENCY_DIR]" >&2 ; + printf -- "%s\n" '---' >&2 ; + printf "%s\n" "Takes a helm repository from INPUT_DIR, with an optional library repository in" >&2 ; + printf "%s\n" "DEPENDENCY_DIR, prepares a view of the git archive at select points in history" >&2 ; + printf "%s\n" "in TMP_DIR and packages helm charts, outputting the tarballs to OUTPUT_DIR" >&2 ; +} + +if [ "x$(basename $PWD)" != "xpackages" ] +then + echo "Error: This script must run from the ./packages/ directory" >&2 + echo >&2 + usage + exit 1 +fi + +if [ "x$#" != "x3" ] && [ "x$#" != "x4" ] +then + echo "Error: This script takes 3 or 4 arguments" >&2 + echo "Got $# arguments:" "$@" >&2 + echo >&2 + usage + exit 1 +fi + +input_dir=$1 +output_dir=$2 +tmp_dir=$3 + +if [ "x$#" = "x4" ] +then + dependency_dir=$4 +fi + +rm -rf "${output_dir:?}" +mkdir -p "${output_dir}" +while read package _ commit +do + # this lets devs build the packages from a dirty repo for quick local testing + if [ "x$commit" = "xHEAD" ] + then + helm package "${input_dir}/${package}" -d "${output_dir}" + continue + fi + git archive --format tar "${commit}" "${input_dir}/${package}" | tar -xf- -C "${tmp_dir}/" + + # the library chart is not present in older commits and git archive doesn't fail gracefully if the path is not found + if [ "x${dependency_dir}" != "x" ] && git ls-tree --name-only "${commit}" "${dependency_dir}" | grep -qx "${dependency_dir}" + then + git archive --format tar "${commit}" "${dependency_dir}" | tar -xf- -C "${tmp_dir}/" + fi + helm package "${tmp_dir}/${input_dir}/${package}" -d "${output_dir}" + rm -rf "${tmp_dir:?}/${input_dir:?}/${package:?}" + if [ "x${dependency_dir}" != "x" ] + then + rm -rf "${tmp_dir:?}/${dependency_dir:?}" + fi +done < "${input_dir}/versions_map" +helm repo index "${output_dir}" diff --git a/packages/apps/Makefile b/packages/apps/Makefile index 506c8750..8751a048 100644 --- a/packages/apps/Makefile +++ b/packages/apps/Makefile @@ -1,14 +1,8 @@ -OUT=../../_out/repos/apps -TMP=../../_out/repos/apps/historical +OUT=../_out/repos/apps +TMP := $(shell mktemp -d) repo: - rm -rf "$(OUT)" - mkdir -p "$(OUT)" - awk '$$3 != "HEAD" {print "mkdir -p $(TMP)/" $$1 "-" $$2}' versions_map | sh -ex - awk '$$3 != "HEAD" {print "git archive " $$3 " " $$1 " | tar -xf- --strip-components=1 -C $(TMP)/" $$1 "-" $$2 }' versions_map | sh -ex - helm package -d "$(OUT)" $$(find . $(TMP) -mindepth 2 -maxdepth 2 -name Chart.yaml | awk 'sub("/Chart.yaml", "")' | sort -V) - cd "$(OUT)" && helm repo index . --url http://cozystack.cozy-system.svc/repos/apps - rm -rf "$(TMP)" + cd .. && ../hack/package_chart.sh apps $(OUT) $(TMP) library fix-chartnames: find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done diff --git a/packages/apps/clickhouse/Chart.yaml b/packages/apps/clickhouse/Chart.yaml index 935cf6af..bfb34b34 100644 --- a/packages/apps/clickhouse/Chart.yaml +++ b/packages/apps/clickhouse/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.8.0 +version: 0.9.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/packages/apps/clickhouse/charts/cozy-lib b/packages/apps/clickhouse/charts/cozy-lib new file mode 120000 index 00000000..e1813509 --- /dev/null +++ b/packages/apps/clickhouse/charts/cozy-lib @@ -0,0 +1 @@ +../../../library/cozy-lib \ No newline at end of file diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index dd4b0fdb..dd97188d 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -122,9 +122,9 @@ spec: - name: clickhouse image: clickhouse/clickhouse-server:24.9.2.42 {{- if .Values.resources }} - resources: {{- toYaml .Values.resources | nindent 16 }} + resources: {{- include "cozy-lib.resources.sanitize" .Values.resources | nindent 16 }} {{- else if ne .Values.resourcesPreset "none" }} - resources: {{- include "resources.preset" (dict "type" .Values.resourcesPreset "Release" .Release) | nindent 16 }} + resources: {{- include "cozy-lib.resources.preset" .Values.resourcesPreset | nindent 16 }} {{- end }} volumeMounts: - name: data-volume-template diff --git a/packages/apps/versions_map b/packages/apps/versions_map index 89d9990d..776c897d 100644 --- a/packages/apps/versions_map +++ b/packages/apps/versions_map @@ -9,7 +9,7 @@ clickhouse 0.6.0 1ec10165 clickhouse 0.6.1 c62a83a7 clickhouse 0.6.2 8267072d clickhouse 0.7.0 93bdf411 -clickhouse 0.8.0 HEAD +clickhouse 0.9.0 HEAD ferretdb 0.1.0 e9716091 ferretdb 0.1.1 91b0499a ferretdb 0.2.0 6c5cf5bf diff --git a/packages/extra/Makefile b/packages/extra/Makefile index 5586c59e..e05e4985 100644 --- a/packages/extra/Makefile +++ b/packages/extra/Makefile @@ -1,14 +1,8 @@ -OUT=../../_out/repos/extra -TMP=../../_out/repos/extra/historical +OUT=../_out/repos/extra +TMP := $(shell mktemp -d) repo: - rm -rf "$(OUT)" - mkdir -p "$(OUT)" - awk '$$3 != "HEAD" {print "mkdir -p $(TMP)/" $$1 "-" $$2}' versions_map | sh -ex - awk '$$3 != "HEAD" {print "git archive " $$3 " " $$1 " | tar -xf- --strip-components=1 -C $(TMP)/" $$1 "-" $$2 }' versions_map | sh -ex - helm package -d "$(OUT)" $$(find . $(TMP) -mindepth 2 -maxdepth 2 -name Chart.yaml | awk 'sub("/Chart.yaml", "")' | sort -V) - cd "$(OUT)" && helm repo index . --url http://cozystack.cozy-system.svc/repos/extra - rm -rf "$(TMP)" + cd .. && ../hack/package_chart.sh extra $(OUT) $(TMP) library fix-chartnames: find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done diff --git a/packages/library/Makefile b/packages/library/Makefile new file mode 100644 index 00000000..664771cf --- /dev/null +++ b/packages/library/Makefile @@ -0,0 +1,15 @@ +OUT=../_out/repos/library +TMP := $(shell mktemp -d) + +repo: + cd .. && ../hack/package_chart.sh library $(OUT) $(TMP) + +fix-chartnames: + find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done + +gen-versions-map: fix-chartnames + ../../hack/gen_versions_map.sh + +check-version-map: gen-versions-map + git diff --exit-code -- versions_map + diff --git a/packages/library/cozy-lib/.helmignore b/packages/library/cozy-lib/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/packages/library/cozy-lib/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/library/cozy-lib/Chart.yaml b/packages/library/cozy-lib/Chart.yaml new file mode 100644 index 00000000..81a3634d --- /dev/null +++ b/packages/library/cozy-lib/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: cozy-lib +description: Common Cozystack templates + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: library + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 diff --git a/packages/library/cozy-lib/Makefile b/packages/library/cozy-lib/Makefile new file mode 100644 index 00000000..fa0142de --- /dev/null +++ b/packages/library/cozy-lib/Makefile @@ -0,0 +1,6 @@ +include ../../../scripts/common-envs.mk +include ../../../scripts/package.mk + +generate: + readme-generator -v values.yaml -s values.schema.json -r README.md + diff --git a/packages/library/cozy-lib/README.md b/packages/library/cozy-lib/README.md new file mode 100644 index 00000000..d4bf18ad --- /dev/null +++ b/packages/library/cozy-lib/README.md @@ -0,0 +1 @@ +## Parameters diff --git a/packages/library/cozy-lib/templates/_resourcepresets.tpl b/packages/library/cozy-lib/templates/_resourcepresets.tpl new file mode 100644 index 00000000..f6ecef3f --- /dev/null +++ b/packages/library/cozy-lib/templates/_resourcepresets.tpl @@ -0,0 +1,49 @@ +{{/* +Copyright Broadcom, Inc. All Rights Reserved. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a resource request/limit object based on a given preset. +These presets are for basic testing and not meant to be used in production +{{ include "cozy-lib.resources.preset" "nano" -}} +*/}} +{{- define "cozy-lib.resources.preset" -}} +{{- $presets := dict + "nano" (dict + "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi") + "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi") + ) + "micro" (dict + "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi") + "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi") + ) + "small" (dict + "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi") + "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi") + ) + "medium" (dict + "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi") + "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi") + ) + "large" (dict + "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi") + "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi") + ) + "xlarge" (dict + "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi") + "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi") + ) + "2xlarge" (dict + "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi") + "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi") + ) + }} +{{- if hasKey $presets . -}} +{{- index $presets . | toYaml -}} +{{- else -}} +{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" . (join "," (keys $presets)) | fail -}} +{{- end -}} +{{- end -}} diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl new file mode 100644 index 00000000..34ed9ca2 --- /dev/null +++ b/packages/library/cozy-lib/templates/_resources.tpl @@ -0,0 +1,53 @@ +{{- /* + A sanitized resource map is a dict with resource-name => resource-quantity. + If not in such a form, requests are used, then limits. All resources are set + to have equal requests and limits, except CPU, that has only requests. The + template expects to receive a dict {"requests":{...}, "limits":{...}} as + input, e.g. {{ include "cozy-lib.resources.sanitize" .Values.resources }}. + Example input: + ============== + limits: + cpu: 100m + memory: 1024Mi + requests: + cpu: 200m + memory: 512Mi + memory: 256Mi + devices.com/nvidia: "1" + + Example output: + =============== + limits: + devices.com/nvidia: "1" + memory: 256Mi + requests: + cpu: 200m + devices.com/nvidia: "1" + memory: 256Mi +*/}} +{{- define "cozy-lib.resources.sanitize" }} +{{- $sanitizedMap := dict }} +{{- if hasKey . "limits" }} +{{- range $k, $v := .limits }} +{{- $_ := set $sanitizedMap $k $v }} +{{- end }} +{{- end }} +{{- if hasKey . "requests" }} +{{- range $k, $v := .requests }} +{{- $_ := set $sanitizedMap $k $v }} +{{- end }} +{{- end }} +{{- range $k, $v := . }} +{{- if not (or (eq $k "requests") (eq $k "limits")) }} +{{- $_ := set $sanitizedMap $k $v }} +{{- end }} +{{- end }} +{{- $output := dict "requests" dict "limits" dict }} +{{- range $k, $v := $sanitizedMap }} +{{- $_ := set $output.requests $k $v }} +{{- if not (eq $k "cpu") }} +{{- $_ := set $output.limits $k $v }} +{{- end }} +{{- end }} +{{- $output | toYaml }} +{{- end }} diff --git a/packages/library/cozy-lib/values.schema.json b/packages/library/cozy-lib/values.schema.json new file mode 100644 index 00000000..decc79aa --- /dev/null +++ b/packages/library/cozy-lib/values.schema.json @@ -0,0 +1,5 @@ +{ + "title": "Chart Values", + "type": "object", + "properties": {} +} \ No newline at end of file diff --git a/packages/library/cozy-lib/values.yaml b/packages/library/cozy-lib/values.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/packages/library/cozy-lib/values.yaml @@ -0,0 +1 @@ +{} diff --git a/packages/library/testfile b/packages/library/testfile new file mode 100644 index 00000000..e69de29b diff --git a/packages/library/versions_map b/packages/library/versions_map new file mode 100644 index 00000000..63834fdd --- /dev/null +++ b/packages/library/versions_map @@ -0,0 +1 @@ +cozy-lib 0.1.0 HEAD