From 9be774ad30fd8d791680ac006e30f0dc059f1173 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 20 Aug 2024 09:07:43 +0200 Subject: [PATCH] Add e2e testing sandbox (#295) This PR introduces new functionality for running e2e-tests in k8s-cluster. `make test` from a root invokes deploying of new sandbox for testing cozystack. from `packages/core/testing`: `make test` - runs the end-to-end tests. `make exec` - opens an interactive shell in the sandbox container. `make login` - downloads the kubeconfig into a temporary directory and runs a shell with the sandbox environment; mirrord must be installed. `make proxy` - enables a SOCKS5 proxy; mirrord and gost must be installed. Signed-off-by: Andrei Kvapil --- Makefile | 5 ++ hack/e2e.sh | 13 +++-- packages/core/testing/Chart.yaml | 3 ++ packages/core/testing/Makefile | 54 +++++++++++++++++++ packages/core/testing/images/e2e-sandbox.json | 10 ++++ .../testing/images/e2e-sandbox/Dockerfile | 15 ++++++ packages/core/testing/templates/sandbox.yaml | 39 ++++++++++++++ packages/core/testing/values.yaml | 2 + 8 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 packages/core/testing/Chart.yaml create mode 100644 packages/core/testing/Makefile create mode 100644 packages/core/testing/images/e2e-sandbox.json create mode 100644 packages/core/testing/images/e2e-sandbox/Dockerfile create mode 100644 packages/core/testing/templates/sandbox.yaml create mode 100644 packages/core/testing/values.yaml diff --git a/Makefile b/Makefile index e424e6d5..62fc1a00 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ build: make -C packages/system/dashboard image make -C packages/system/kamaji image make -C packages/core/installer image + make -C packages/core/e2e image make manifests manifests: @@ -26,3 +27,7 @@ repos: assets: make -C packages/core/installer/ assets + +test: + make -C packages/core/testing apply + make -C packages/core/testing test diff --git a/hack/e2e.sh b/hack/e2e.sh index 462fef65..4d8329c9 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -27,9 +27,9 @@ ip link add cozy-br0 type bridge ip link set cozy-br0 up ip addr add 192.168.123.1/24 dev cozy-br0 -# Enable forward & masquerading -echo 1 > /proc/sys/net/ipv4/ip_forward -iptables -t nat -A POSTROUTING -s 192.168.123.0/24 -j MASQUERADE +# Enable masquerading +iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE 2>/dev/null || true +iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE rm -rf srv1 srv2 srv3 mkdir -p srv1 srv2 srv3 @@ -287,7 +287,8 @@ kubectl patch -n tenant-root hr/tenant-root --type=merge -p '{"spec":{ "values": "host": "example.org", "ingress": true, "monitoring": true, - "etcd": true + "etcd": true, + "isolated": true }}}' # Wait for HelmRelease be created @@ -296,6 +297,10 @@ timeout 60 sh -c 'until kubectl get hr -n tenant-root etcd ingress monitoring te # Wait for HelmReleases be installed kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr etcd ingress monitoring tenant-root +kubectl patch -n tenant-root hr/ingress --type=merge -p '{"spec":{ "values":{ + "dashboard": true +}}}' + # Wait for nginx-ingress-controller timeout 60 sh -c 'until kubectl get deploy -n tenant-root root-ingress-controller; do sleep 1; done' kubectl wait --timeout=5m --for=condition=available -n tenant-root deploy root-ingress-controller diff --git a/packages/core/testing/Chart.yaml b/packages/core/testing/Chart.yaml new file mode 100644 index 00000000..293103dc --- /dev/null +++ b/packages/core/testing/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: e2e +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/core/testing/Makefile b/packages/core/testing/Makefile new file mode 100644 index 00000000..b0ba2964 --- /dev/null +++ b/packages/core/testing/Makefile @@ -0,0 +1,54 @@ +NAMESPACE=cozy-e2e-tests +NAME := sandbox +CLEAN := 1 + +include ../../../scripts/common-envs.mk + + +help: ## Show this help. + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) +show: + helm template -n $(NAMESPACE) $(NAME) . + +apply: ## Create sandbox in existing Kubernetes cluster. + helm template -n $(NAMESPACE) $(NAME) . | kubectl apply -f - + +diff: + helm template -n $(NAMESPACE) $(NAME) . | kubectl diff -f - + +image: image-e2e-sandbox + +image-e2e-sandbox: + docker buildx build -f images/e2e-sandbox/Dockerfile ../../.. \ + --provenance false \ + --tag $(REGISTRY)/e2e-sandbox:$(call settag,$(TAG)) \ + --cache-from type=registry,ref=$(REGISTRY)/e2e-sandbox:latest \ + --platform linux/amd64,linux/arm64 \ + --cache-to type=inline \ + --metadata-file images/e2e-sandbox.json \ + --push=$(PUSH) \ + --load=$(LOAD) + IMAGE="$(REGISTRY)/e2e-sandbox:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/e2e-sandbox.json -o json -r)" \ + yq -i '.e2e.image = strenv(IMAGE)' values.yaml + +test: ## Run the end-to-end tests in existing sandbox. + kubectl wait deploy --for=condition=Progressing -n $(NAMESPACE) cozystack-e2e-$(NAME) + sleep 3 + cat ../../../hack/e2e.sh | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'cat > /e2e.sh && chmod +x /e2e.sh' + helm template -n cozy-system installer ../installer | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'cat > /cozystack-installer.yaml' + kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'export COZYSTACK_INSTALLER_YAML=$$(cat /cozystack-installer.yaml) && /e2e.sh' + +delete: ## Remove sandbox from existing Kubernetes cluster. + kubectl delete deploy -n $(NAMESPACE) cozystack-e2e-$(NAME) + +exec: ## Opens an interactive shell in the sandbox container. + kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- bash + +proxy: sync-hosts ## Enable a SOCKS5 proxy server; mirrord and gost must be installed. + mirrord exec --target deploy/cozystack-e2e-sandbox --target-namespace cozy-e2e-tests -- gost -L=127.0.0.1:10080 + +login: ## Downloads the kubeconfig into a temporary directory and runs a shell with the sandbox environment; mirrord must be installed. + mirrord exec --target deploy/cozystack-e2e-sandbox --target-namespace cozy-e2e-tests -- "$$SHELL" + +sync-hosts: + kubectl exec -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'kubectl get ing -A -o go-template='\''{{ "127.0.0.1 localhost\n"}}{{ range .items }}{{ range .status.loadBalancer.ingress }}{{ .ip }}{{ end }} {{ range .spec.rules }}{{ .host }}{{ end }}{{ "\n" }}{{ end }}'\'' > /etc/hosts' diff --git a/packages/core/testing/images/e2e-sandbox.json b/packages/core/testing/images/e2e-sandbox.json new file mode 100644 index 00000000..647d660b --- /dev/null +++ b/packages/core/testing/images/e2e-sandbox.json @@ -0,0 +1,10 @@ +{ + "buildx.build.ref": "buildkit/buildkit0/yej19zqlualmgypt6eig19jvv", + "containerimage.descriptor": { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "digest": "sha256:be1693c8ce6a9522499f79b1e42b2e08c7ca80405026a095299e5e990a3ab791", + "size": 685 + }, + "containerimage.digest": "sha256:be1693c8ce6a9522499f79b1e42b2e08c7ca80405026a095299e5e990a3ab791", + "image.name": "ghcr.io/aenix-io/cozystack/e2e-sandbox:latest" +} \ No newline at end of file diff --git a/packages/core/testing/images/e2e-sandbox/Dockerfile b/packages/core/testing/images/e2e-sandbox/Dockerfile new file mode 100644 index 00000000..a07dccd3 --- /dev/null +++ b/packages/core/testing/images/e2e-sandbox/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:22.04 + +ARG KUBECTL_VERSION=1.31.0 +ARG TALOSCTL_VERSION=1.7.6 +ARG HELM_VERSION=3.15.4 + +RUN apt-get update +RUN apt-get -y install genisoimage qemu-kvm qemu-utils iproute2 iptables wget xz-utils netcat curl +RUN curl -LO "https://github.com/siderolabs/talos/releases/download/v${TALOSCTL_VERSION}/talosctl-linux-amd64" \ + && chmod +x talosctl-linux-amd64 \ + && mv talosctl-linux-amd64 /usr/local/bin/talosctl +RUN curl -LO "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/kubectl +RUN curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -s - --version "v${HELM_VERSION}" diff --git a/packages/core/testing/templates/sandbox.yaml b/packages/core/testing/templates/sandbox.yaml new file mode 100644 index 00000000..d909b297 --- /dev/null +++ b/packages/core/testing/templates/sandbox.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} + labels: + pod-security.kubernetes.io/enforce: privileged +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cozystack-e2e-{{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app: cozystack-e2e-{{ .Release.Name }} + strategy: + type: Recreate + template: + metadata: + labels: + app: cozystack-e2e-{{ .Release.Name }} + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 1 + containers: + - name: sandbox + image: "{{ .Values.e2e.image }}" + securityContext: + privileged: true + env: + - name: KUBECONFIG + value: /kubeconfig + - name: TALOSCONFIG + value: /talosconfig + command: + - sleep + - infinity diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml new file mode 100644 index 00000000..10e5edef --- /dev/null +++ b/packages/core/testing/values.yaml @@ -0,0 +1,2 @@ +e2e: + image: ghcr.io/aenix-io/cozystack/e2e-sandbox:latest@sha256:be1693c8ce6a9522499f79b1e42b2e08c7ca80405026a095299e5e990a3ab791