Add e2e testing sandbox (#295)

This PR introduces new functionality for running e2e-tests in k8s-cluster. `make test` from a root invokes deploying of new sandbox for testing cozystack. from `packages/core/testing`: `make test` - runs the end-to-end tests. `make exec` - opens an interactive shell in the sandbox container. `make login` - downloads the kubeconfig into a temporary directory and runs a shell with the sandbox environment; mirrord must be installed. `make proxy` - enables a SOCKS5 proxy; mirrord and gost must be installed. Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
2026-01-27 10:18:39 +00:00 · 2024-08-20 09:07:43 +02:00
parent 3b67f1fb27
commit 9be774ad30
8 changed files with 137 additions and 4 deletions
--- a/5
+++ b/5
@@ -8,6 +8,7 @@ build:
 	make -C packages/system/dashboard image
 	make -C packages/system/kamaji image
 	make -C packages/core/installer image
+	make -C packages/core/e2e image
 	make manifests

 manifests:
@@ -26,3 +27,7 @@ repos:

 assets:
 	make -C packages/core/installer/ assets
+
+test:
+	make -C packages/core/testing apply
+	make -C packages/core/testing test
--- a/hack/e2e.sh
+++ b/hack/e2e.sh
@@ -27,9 +27,9 @@ ip link add cozy-br0 type bridge
 ip link set cozy-br0 up
 ip addr add 192.168.123.1/24 dev cozy-br0

-# Enable forward & masquerading
-echo 1 > /proc/sys/net/ipv4/ip_forward
-iptables -t nat -A POSTROUTING -s 192.168.123.0/24 -j MASQUERADE
+# Enable masquerading
+iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE 2>/dev/null || true
+iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE

 rm -rf srv1 srv2 srv3
 mkdir -p srv1 srv2 srv3
@@ -287,7 +287,8 @@ kubectl patch -n tenant-root hr/tenant-root --type=merge -p '{"spec":{ "values":
  "host": "example.org",
  "ingress": true,
  "monitoring": true,
-  "etcd": true
+  "etcd": true,
+  "isolated": true
 }}}'

 # Wait for HelmRelease be created
@@ -296,6 +297,10 @@ timeout 60 sh -c 'until kubectl get hr -n tenant-root etcd ingress monitoring te
 # Wait for HelmReleases be installed
 kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr etcd ingress monitoring tenant-root

+kubectl patch -n tenant-root hr/ingress --type=merge -p '{"spec":{ "values":{
+  "dashboard": true
+}}}'
+
 # Wait for nginx-ingress-controller
 timeout 60 sh -c 'until kubectl get deploy -n tenant-root root-ingress-controller; do sleep 1; done'
 kubectl wait --timeout=5m --for=condition=available -n tenant-root deploy root-ingress-controller
--- a/packages/core/testing/Chart.yaml
+++ b/packages/core/testing/Chart.yaml
@@ -0,0 +1,3 @@
+apiVersion: v2
+name: e2e
+version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process
--- a/packages/core/testing/Makefile
+++ b/packages/core/testing/Makefile
@@ -0,0 +1,54 @@
+NAMESPACE=cozy-e2e-tests
+NAME := sandbox
+CLEAN := 1
+
+include ../../../scripts/common-envs.mk
+
+
+help: ## Show this help.
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+show:
+	helm template -n $(NAMESPACE) $(NAME) .
+
+apply: ## Create sandbox in existing Kubernetes cluster.
+	helm template -n $(NAMESPACE) $(NAME) . | kubectl apply -f -
+
+diff:
+	helm template -n $(NAMESPACE) $(NAME) . | kubectl diff -f -
+
+image: image-e2e-sandbox
+
+image-e2e-sandbox:
+	docker buildx build -f images/e2e-sandbox/Dockerfile ../../.. \
+		--provenance false \
+		--tag $(REGISTRY)/e2e-sandbox:$(call settag,$(TAG)) \
+		--cache-from type=registry,ref=$(REGISTRY)/e2e-sandbox:latest \
+		--platform linux/amd64,linux/arm64 \
+		--cache-to type=inline \
+		--metadata-file images/e2e-sandbox.json \
+		--push=$(PUSH) \
+		--load=$(LOAD)
+	IMAGE="$(REGISTRY)/e2e-sandbox:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/e2e-sandbox.json -o json -r)" \
+		yq -i '.e2e.image = strenv(IMAGE)' values.yaml
+
+test: ## Run the end-to-end tests in existing sandbox.
+	kubectl wait deploy --for=condition=Progressing -n $(NAMESPACE) cozystack-e2e-$(NAME)
+	sleep 3
+	cat ../../../hack/e2e.sh | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'cat > /e2e.sh && chmod +x /e2e.sh'
+	helm template -n cozy-system installer ../installer | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'cat > /cozystack-installer.yaml'
+	kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'export COZYSTACK_INSTALLER_YAML=$$(cat /cozystack-installer.yaml) && /e2e.sh'
+
+delete: ## Remove sandbox from existing Kubernetes cluster.
+	kubectl delete deploy -n $(NAMESPACE) cozystack-e2e-$(NAME)
+
+exec:  ## Opens an interactive shell in the sandbox container.
+	kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- bash
+
+proxy: sync-hosts ## Enable a SOCKS5 proxy server; mirrord and gost must be installed.
+	mirrord exec --target deploy/cozystack-e2e-sandbox --target-namespace cozy-e2e-tests -- gost -L=127.0.0.1:10080
+
+login: ## Downloads the kubeconfig into a temporary directory and runs a shell with the sandbox environment; mirrord must be installed.
+	mirrord exec --target deploy/cozystack-e2e-sandbox --target-namespace cozy-e2e-tests -- "$$SHELL"
+
+sync-hosts:
+	kubectl exec -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'kubectl get ing -A -o go-template='\''{{ "127.0.0.1 localhost\n"}}{{ range .items }}{{ range .status.loadBalancer.ingress }}{{ .ip }}{{ end }} {{ range .spec.rules }}{{ .host }}{{ end }}{{ "\n" }}{{ end }}'\'' > /etc/hosts'
--- a/packages/core/testing/images/e2e-sandbox.json
+++ b/packages/core/testing/images/e2e-sandbox.json
@@ -0,0 +1,10 @@
+{
+  "buildx.build.ref": "buildkit/buildkit0/yej19zqlualmgypt6eig19jvv",
+  "containerimage.descriptor": {
+    "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
+    "digest": "sha256:be1693c8ce6a9522499f79b1e42b2e08c7ca80405026a095299e5e990a3ab791",
+    "size": 685
+  },
+  "containerimage.digest": "sha256:be1693c8ce6a9522499f79b1e42b2e08c7ca80405026a095299e5e990a3ab791",
+  "image.name": "ghcr.io/aenix-io/cozystack/e2e-sandbox:latest"
+}
--- a/packages/core/testing/images/e2e-sandbox/Dockerfile
+++ b/packages/core/testing/images/e2e-sandbox/Dockerfile
@@ -0,0 +1,15 @@
+FROM ubuntu:22.04
+
+ARG KUBECTL_VERSION=1.31.0
+ARG TALOSCTL_VERSION=1.7.6
+ARG HELM_VERSION=3.15.4
+
+RUN apt-get update
+RUN apt-get -y install genisoimage qemu-kvm qemu-utils iproute2 iptables wget xz-utils netcat curl
+RUN curl -LO "https://github.com/siderolabs/talos/releases/download/v${TALOSCTL_VERSION}/talosctl-linux-amd64" \
+ && chmod +x talosctl-linux-amd64 \
+ && mv talosctl-linux-amd64 /usr/local/bin/talosctl
+RUN curl -LO "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
+ && chmod +x kubectl \
+ && mv kubectl /usr/local/bin/kubectl
+RUN curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -s - --version "v${HELM_VERSION}"
--- a/packages/core/testing/templates/sandbox.yaml
+++ b/packages/core/testing/templates/sandbox.yaml
@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ .Release.Namespace }}
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cozystack-e2e-{{ .Release.Name }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cozystack-e2e-{{ .Release.Name }}
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: cozystack-e2e-{{ .Release.Name }}
+    spec:
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 1
+      containers:
+      - name: sandbox
+        image: "{{ .Values.e2e.image }}"
+        securityContext:
+          privileged: true
+        env:
+        - name: KUBECONFIG
+          value: /kubeconfig
+        - name: TALOSCONFIG
+          value: /talosconfig
+        command:
+        - sleep
+        - infinity
--- a/packages/core/testing/values.yaml
+++ b/packages/core/testing/values.yaml
@@ -0,0 +1,2 @@
+e2e:
+  image: ghcr.io/aenix-io/cozystack/e2e-sandbox:latest@sha256:be1693c8ce6a9522499f79b1e42b2e08c7ca80405026a095299e5e990a3ab791