From eedc4ebce158be35953516838435e72de8f6f506 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 12 Jun 2024 19:19:14 +0200 Subject: [PATCH 1/6] Add e2e tests Signed-off-by: Andrei Kvapil --- hack/e2e.sh | 305 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100755 hack/e2e.sh diff --git a/hack/e2e.sh b/hack/e2e.sh new file mode 100755 index 00000000..90c8314d --- /dev/null +++ b/hack/e2e.sh @@ -0,0 +1,305 @@ +#!/bin/bash +if [ "$COZYSTACK_INSTALLER_YAML" = "" ]; then + echo 'COZYSTACK_INSTALLER_YAML variable is not set!' >&2 + echo 'please set it with following command:' >&2 + echo >&2 + echo 'export COZYSTACK_INSTALLER_YAML=$(helm template -n cozy-system installer packages/core/installer)' >&2 + echo >&2 + exit 1 +fi + +set -x +set -e + +kill `cat srv1/qemu.pid srv2/qemu.pid srv3/qemu.pid` || true + +ip link del cozy-br0 || true +ip link add cozy-br0 type bridge +ip link set cozy-br0 up +ip addr add 192.168.123.1/24 dev cozy-br0 + +rm -rf srv1 srv2 srv3 +mkdir -p srv1 srv2 srv3 + +# Prepare cloud-init +for i in 1 2 3; do + echo "local-hostname: srv$i" > "srv$i/meta-data" + echo '#cloud-config' > "srv$i/user-data" + cat > "srv$i/network-config" < patch.yaml <<\EOT +machine: + kubelet: + nodeIP: + validSubnets: + - 192.168.123.0/24 + extraConfig: + maxPods: 512 + kernel: + modules: + - name: openvswitch + - name: drbd + parameters: + - usermode_helper=disabled + - name: zfs + - name: spl + install: + image: ghcr.io/aenix-io/cozystack/talos:v1.7.1 + files: + - content: | + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + device_ownership_from_security_context = true + path: /etc/cri/conf.d/20-customization.part + op: create + +cluster: + network: + cni: + name: none + dnsDomain: cozy.local + podSubnets: + - 10.244.0.0/16 + serviceSubnets: + - 10.96.0.0/16 +EOT + +cat > patch-controlplane.yaml <<\EOT +machine: + network: + interfaces: + - interface: eth0 + vip: + ip: 192.168.123.10 +cluster: + allowSchedulingOnControlPlanes: true + controllerManager: + extraArgs: + bind-address: 0.0.0.0 + scheduler: + extraArgs: + bind-address: 0.0.0.0 + apiServer: + certSANs: + - 127.0.0.1 + proxy: + disabled: true + discovery: + enabled: false + etcd: + advertisedSubnets: + - 192.168.123.0/24 +EOT + +# Gen configuration +if [ ! -f secrets.yaml ]; then + talosctl gen secrets +fi + +rm -f controlplane.yaml worker.yaml talosconfig kubeconfig +talosctl gen config --with-secrets secrets.yaml cozystack https://192.168.123.10:6443 --config-patch=@patch.yaml --config-patch-control-plane @patch-controlplane.yaml +export TALOSCONFIG=$PWD/talosconfig + +# Apply configuration +talosctl apply -f controlplane.yaml -n 192.168.123.11 -e 192.168.123.11 -i +talosctl apply -f controlplane.yaml -n 192.168.123.12 -e 192.168.123.12 -i +talosctl apply -f controlplane.yaml -n 192.168.123.13 -e 192.168.123.13 -i + +# Wait for VM to be configured +timeout 60 sh -c 'until nc -nzv 192.168.123.11 50000 && nc -nzv 192.168.123.12 50000 && nc -nzv 192.168.123.13 50000; do sleep 1; done' + +# Bootstrap +talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11 + +# Wait for etcd +timeout 120 sh -c 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1 | grep "rpc error"; do sleep 1; done' + +rm -f kubeconfig +talosctl kubeconfig kubeconfig -e 192.168.123.10 -n 192.168.123.10 +export KUBECONFIG=$PWD/kubeconfig + +# Wait for kubernetes nodes appear +timeout 60 sh -c 'until [ $(kubectl get node -o name | wc -l) = 3 ]; do sleep 1; done' +kubectl create ns cozy-system +kubectl create -f - <<\EOT +apiVersion: v1 +kind: ConfigMap +metadata: + name: cozystack + namespace: cozy-system +data: + bundle-name: "paas-full" + ipv4-pod-cidr: "10.244.0.0/16" + ipv4-pod-gateway: "10.244.0.1" + ipv4-svc-cidr: "10.96.0.0/16" + ipv4-join-cidr: "100.64.0.0/16" +EOT + +# +echo "$COZYSTACK_INSTALLER_YAML" | kubectl apply -f - + +# wait for cozystack pod to start +kubectl wait deploy --timeout=1m --for=condition=available -n cozy-system cozystack + +# wait for helmreleases appear +timeout 60 sh -c 'until kubectl get hr -A | grep cozy; do sleep 1; done' + +sleep 5 + +kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n " $1 " hr/" $2 " &"} END{print "wait"}' | sh -x +# Wait for linstor controller +kubectl wait deploy --timeout=5m --for=condition=available -n cozy-linstor linstor-controller + +# Wait for all linstor nodes become Online +timeout 60 sh -c 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) = 3 ]; do sleep 1; done' + +kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs srv1 /dev/vdc --pool-name data --storage-pool data +kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs srv2 /dev/vdc --pool-name data --storage-pool data +kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs srv3 /dev/vdc --pool-name data --storage-pool data + +kubectl create -f- < Date: Mon, 17 Jun 2024 17:37:24 +0200 Subject: [PATCH 2/6] add check for forwarding and masquerading Signed-off-by: Andrei Kvapil --- hack/e2e.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hack/e2e.sh b/hack/e2e.sh index 90c8314d..db86d04f 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -8,6 +8,15 @@ if [ "$COZYSTACK_INSTALLER_YAML" = "" ]; then exit 1 fi +if [ "$(cat /proc/sys/net/ipv4/ip_forward)" != 1 ]; then + echo "IPv4 forwarding is not enabled!" >&2 + echo 'please enable forwarding with the following command:' >&2 + echo >&2 + echo 'echo 1 > /proc/sys/net/ipv4/ip_forward' >&2 + echo >&2 + exit 1 +fi + set -x set -e @@ -18,6 +27,10 @@ ip link add cozy-br0 type bridge ip link set cozy-br0 up ip addr add 192.168.123.1/24 dev cozy-br0 +# Enable masquerading +iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE || true +iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE + rm -rf srv1 srv2 srv3 mkdir -p srv1 srv2 srv3 From bcd1ee1b4f7e5c0a21bdbc4392a74387bba5852c Mon Sep 17 00:00:00 2001 From: Marian Koreniuk Date: Mon, 17 Jun 2024 19:13:54 +0200 Subject: [PATCH 3/6] Add masquerade --- hack/e2e.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/e2e.sh b/hack/e2e.sh index db86d04f..86278beb 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -27,9 +27,9 @@ ip link add cozy-br0 type bridge ip link set cozy-br0 up ip addr add 192.168.123.1/24 dev cozy-br0 -# Enable masquerading -iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE || true -iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE +# Enable forward & masquerading +echo 1 > /proc/sys/net/ipv4/ip_forward +iptables -t nat -A POSTROUTING -s 192.168.123.0/24 -j MASQUERADE rm -rf srv1 srv2 srv3 mkdir -p srv1 srv2 srv3 From 02a41e126bd1927fda7e395fcb6d81e5fcd12939 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 19 Jun 2024 16:55:16 +0200 Subject: [PATCH 4/6] fix kubeovn and cilium tags (#174) * fix: kube-ovn tag * fix: cilium tag --- packages/system/cilium/Makefile | 2 +- packages/system/cilium/images/cilium.tag | 2 +- packages/system/kubeovn/Makefile | 2 +- packages/system/kubeovn/images/kubeovn.tag | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/system/cilium/Makefile b/packages/system/cilium/Makefile index 23e43516..a5caba3b 100644 --- a/packages/system/cilium/Makefile +++ b/packages/system/cilium/Makefile @@ -27,4 +27,4 @@ image: --metadata-file images/cilium.json \ --push=$(PUSH) \ --load=$(LOAD) - echo "$(REGISTRY)/cilium:$(call settag,$(TAG))" > images/cilium.tag + echo "$(REGISTRY)/cilium:$(call settag,$(CILIUM_TAG))" > images/cilium.tag diff --git a/packages/system/cilium/images/cilium.tag b/packages/system/cilium/images/cilium.tag index 164d288a..ab03ee07 100644 --- a/packages/system/cilium/images/cilium.tag +++ b/packages/system/cilium/images/cilium.tag @@ -1 +1 @@ -ghcr.io/aenix-io/cozystack/cilium:v0.7.0 +ghcr.io/aenix-io/cozystack/cilium:latest diff --git a/packages/system/kubeovn/Makefile b/packages/system/kubeovn/Makefile index d4e6884e..783005ba 100644 --- a/packages/system/kubeovn/Makefile +++ b/packages/system/kubeovn/Makefile @@ -24,4 +24,4 @@ image: --metadata-file images/kubeovn.json \ --push=$(PUSH) \ --load=$(LOAD) - echo "$(REGISTRY)/kubeovn:$(call settag,$(TAG))" > images/kubeovn.tag + echo "$(REGISTRY)/kubeovn:$(call settag,$(KUBEOVN_TAG))" > images/kubeovn.tag diff --git a/packages/system/kubeovn/images/kubeovn.tag b/packages/system/kubeovn/images/kubeovn.tag index 2236b16e..f5b70b27 100644 --- a/packages/system/kubeovn/images/kubeovn.tag +++ b/packages/system/kubeovn/images/kubeovn.tag @@ -1 +1 @@ -ghcr.io/aenix-io/cozystack/kubeovn:v0.7.0 +ghcr.io/aenix-io/cozystack/kubeovn:v1.13.0 From f12e2c300a8b824726f790d2150dd943b2b44a50 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Thu, 20 Jun 2024 13:43:04 +0200 Subject: [PATCH 5/6] add initial arm support Signed-off-by: Andrei Kvapil --- packages/core/installer/Makefile | 1 + packages/core/installer/images/cozystack.json | 10 ++++++++-- packages/core/installer/images/cozystack.tag | 2 +- packages/core/installer/images/cozystack/Dockerfile | 5 ++++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/core/installer/Makefile b/packages/core/installer/Makefile index 7cea7ed7..090cb4b0 100644 --- a/packages/core/installer/Makefile +++ b/packages/core/installer/Makefile @@ -25,6 +25,7 @@ image-cozystack: --provenance false \ --tag $(REGISTRY)/cozystack:$(call settag,$(TAG)) \ --cache-from type=registry,ref=$(REGISTRY)/cozystack:latest \ + --platform linux/amd64,linux/arm64 \ --cache-to type=inline \ --metadata-file images/cozystack.json \ --push=$(PUSH) \ diff --git a/packages/core/installer/images/cozystack.json b/packages/core/installer/images/cozystack.json index 585f11e3..63a732c3 100644 --- a/packages/core/installer/images/cozystack.json +++ b/packages/core/installer/images/cozystack.json @@ -1,4 +1,10 @@ { - "containerimage.config.digest": "sha256:6d54a5b971e80fbaace664054d4e67f24fd1fbb7807ebaffd036d4ea7195df10", - "containerimage.digest": "sha256:a6b167235d8556ff7e45f4582c2491a2ad48292a46005dcf767908e2fb78e74e" + "buildx.build.ref": "youthful_hertz/youthful_hertz0/aafwjh8j28i98f59smgh3qe86", + "containerimage.descriptor": { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "digest": "sha256:e0c0defb9f5b10f5187d4002ccec7d01841e96c7350963f253003c0efeff6cef", + "size": 685 + }, + "containerimage.digest": "sha256:e0c0defb9f5b10f5187d4002ccec7d01841e96c7350963f253003c0efeff6cef", + "image.name": "ghcr.io/aenix-io/cozystack/cozystack:latest" } \ No newline at end of file diff --git a/packages/core/installer/images/cozystack.tag b/packages/core/installer/images/cozystack.tag index e8430b50..602d7b26 100644 --- a/packages/core/installer/images/cozystack.tag +++ b/packages/core/installer/images/cozystack.tag @@ -1 +1 @@ -ghcr.io/aenix-io/cozystack/cozystack:v0.7.0 +ghcr.io/aenix-io/cozystack/cozystack:latest diff --git a/packages/core/installer/images/cozystack/Dockerfile b/packages/core/installer/images/cozystack/Dockerfile index 6618cd00..7c9652b3 100644 --- a/packages/core/installer/images/cozystack/Dockerfile +++ b/packages/core/installer/images/cozystack/Dockerfile @@ -3,12 +3,15 @@ FROM golang:alpine3.19 as k8s-await-election-builder ARG K8S_AWAIT_ELECTION_GITREPO=https://github.com/LINBIT/k8s-await-election ARG K8S_AWAIT_ELECTION_VERSION=0.4.1 +# TARGETARCH is a docker special variable: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +ARG TARGETARCH + RUN apk add --no-cache git make RUN git clone ${K8S_AWAIT_ELECTION_GITREPO} /usr/local/go/k8s-await-election/ \ && cd /usr/local/go/k8s-await-election \ && git reset --hard v${K8S_AWAIT_ELECTION_VERSION} \ && make \ - && mv ./out/k8s-await-election-amd64 /k8s-await-election + && mv ./out/k8s-await-election-${TARGETARCH} /k8s-await-election FROM alpine:3.19 AS builder From 995dea6f5cbec81990eb302574b23af6bdb34511 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Fri, 21 Jun 2024 10:10:11 +0200 Subject: [PATCH 6/6] postgres: option to enable quorum-based replication Signed-off-by: Andrei Kvapil --- packages/apps/postgres/Chart.yaml | 2 +- packages/apps/postgres/README.md | 12 +++++++----- packages/apps/postgres/templates/db.yaml | 3 +++ packages/apps/postgres/values.schema.json | 17 ++++++++++++++++- packages/apps/postgres/values.yaml | 9 ++++++++- packages/apps/versions_map | 3 ++- 6 files changed, 37 insertions(+), 9 deletions(-) diff --git a/packages/apps/postgres/Chart.yaml b/packages/apps/postgres/Chart.yaml index 89dee73f..eb9ab8cc 100644 --- a/packages/apps/postgres/Chart.yaml +++ b/packages/apps/postgres/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.1 +version: 0.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 5bdf290f..6fbfc582 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -35,11 +35,13 @@ more details: ### Common parameters -| Name | Description | Value | -| ---------- | ----------------------------------------------- | ------- | -| `external` | Enable external access from outside the cluster | `false` | -| `size` | Persistent Volume size | `10Gi` | -| `replicas` | Number of MariaDB replicas | `2` | +| Name | Description | Value | +| ------------------------ | ----------------------------------------------------------------------------------------------------------------------- | ------- | +| `external` | Enable external access from outside the cluster | `false` | +| `size` | Persistent Volume size | `10Gi` | +| `replicas` | Number of Postgres replicas | `2` | +| `quorum.minSyncReplicas` | Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed. | `0` | +| `quorum.maxSyncReplicas` | Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances). | `0` | ### Configuration parameters diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 7c7d5b9e..7c1483a1 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -11,6 +11,9 @@ spec: parameters: max_wal_senders: "30" + minSyncReplicas: {{ .Values.quorum.minSyncReplicas }} + maxSyncReplicas: {{ .Values.quorum.maxSyncReplicas }} + monitoring: enablePodMonitor: true diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 11e09d44..51b81336 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -14,9 +14,24 @@ }, "replicas": { "type": "number", - "description": "Number of MariaDB replicas", + "description": "Number of Postgres replicas", "default": 2 }, + "quorum": { + "type": "object", + "properties": { + "minSyncReplicas": { + "type": "number", + "description": "Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed.", + "default": 0 + }, + "maxSyncReplicas": { + "type": "number", + "description": "Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances).", + "default": 0 + } + } + }, "databases": { "type": "object", "description": "Databases configuration", diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index 2b7b49c1..0fdd8c19 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -2,12 +2,19 @@ ## @param external Enable external access from outside the cluster ## @param size Persistent Volume size -## @param replicas Number of MariaDB replicas +## @param replicas Number of Postgres replicas ## external: false size: 10Gi replicas: 2 +## Configuration for the quorum-based synchronous replication +## @param quorum.minSyncReplicas Minimum number of synchronous replicas that must acknowledge a transaction before it is considered committed. +## @param quorum.maxSyncReplicas Maximum number of synchronous replicas that can acknowledge a transaction (must be lower than the number of instances). +quorum: + minSyncReplicas: 0 + maxSyncReplicas: 0 + ## @section Configuration parameters ## @param users [object] Users configuration diff --git a/packages/apps/versions_map b/packages/apps/versions_map index 712c3014..5d5d46b8 100644 --- a/packages/apps/versions_map +++ b/packages/apps/versions_map @@ -14,7 +14,8 @@ mysql 0.2.0 8b975ff0 mysql 0.3.0 HEAD postgres 0.1.0 f642698 postgres 0.2.0 7cd7de73 -postgres 0.2.1 HEAD +postgres 0.2.1 4a97e297 +postgres 0.3.0 HEAD rabbitmq 0.1.0 f642698 rabbitmq 0.2.0 HEAD redis 0.1.1 f642698