From 6980dc59c512fe9a529fb52d3e08e1c3f433e478 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Thu, 3 Apr 2025 10:49:01 +0200 Subject: [PATCH] Add workflow to run e2e tests using GitHub CI Signed-off-by: Andrei Kvapil --- .github/workflows/pull-requests.yaml | 37 +++++ .github/workflows/tags.yaml | 140 ++++++++++++++++++ Makefile | 12 +- hack/e2e.sh | 2 +- packages/core/builder/Chart.yaml | 3 - packages/core/builder/Makefile | 35 ----- packages/core/builder/config.toml | 11 -- packages/core/builder/templates/sandbox.yaml | 43 ------ packages/core/builder/values.yaml | 3 - packages/core/installer/Makefile | 17 +-- packages/core/testing/Makefile | 31 ++-- .../testing/images/e2e-sandbox/Dockerfile | 4 +- packages/core/testing/templates/sandbox.yaml | 40 ----- packages/core/testing/values.yaml | 2 +- 14 files changed, 208 insertions(+), 172 deletions(-) create mode 100644 .github/workflows/pull-requests.yaml create mode 100644 .github/workflows/tags.yaml delete mode 100755 packages/core/builder/Chart.yaml delete mode 100755 packages/core/builder/Makefile delete mode 100644 packages/core/builder/config.toml delete mode 100755 packages/core/builder/templates/sandbox.yaml delete mode 100755 packages/core/builder/values.yaml delete mode 100755 packages/core/testing/templates/sandbox.yaml diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml new file mode 100644 index 00000000..7cc92562 --- /dev/null +++ b/.github/workflows/pull-requests.yaml @@ -0,0 +1,37 @@ +name: Build and Test + +on: + pull_request: + types: [labeled, opened, synchronize, reopened] + +jobs: + e2e: + name: Build and Test + runs-on: [self-hosted] + permissions: + contents: read + packages: write + + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + + - name: make build + run: | + make build + + - name: make test + run: | + make test diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml new file mode 100644 index 00000000..7f5d975f --- /dev/null +++ b/.github/workflows/tags.yaml @@ -0,0 +1,140 @@ +name: Prepare Release + +on: + push: + tags: + - 'v*.*.*' + +jobs: + prepare-release: + name: Build, Test and Prepare Release + runs-on: [self-hosted] + permissions: + contents: write + packages: write + + steps: + - name: Check if release already exists + id: check_release + uses: actions/github-script@v7 + with: + script: | + const tag = context.ref.replace('refs/tags/', ''); + const releases = await github.rest.repos.listReleases({ + owner: context.repo.owner, + repo: context.repo.repo + }); + + const existing = releases.data.find(r => r.tag_name === tag && !r.draft); + if (existing) { + core.setOutput('skip', 'true'); + } else { + core.setOutput('skip', 'false'); + } + + - name: Skip if release already exists + if: steps.check_release.outputs.skip == 'true' + run: echo "Release already exists, skipping workflow." + + - name: Checkout code + if: steps.check_release.outputs.skip == 'false' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Login to GitHub Container Registry + if: steps.check_release.outputs.skip == 'false' + uses: docker/login-action@v3 + with: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + + - name: Build + if: steps.check_release.outputs.skip == 'false' + run: make build + + #- name: Test + # if: steps.check_release.outputs.skip == 'false' + # run: make test + + - name: Commit release artifacts + if: steps.check_release.outputs.skip == 'false' + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + git add . + git commit -m "Prepare release ${GITHUB_REF#refs/tags/}" || echo "No changes to commit" + + - name: Create release branch + if: steps.check_release.outputs.skip == 'false' + run: | + BRANCH_NAME="release-${GITHUB_REF#refs/tags/v}" + git branch -f "$BRANCH_NAME" + git push origin "$BRANCH_NAME" --force + + - name: Create pull request if not exists + if: steps.check_release.outputs.skip == 'false' + uses: actions/github-script@v7 + with: + script: | + const version = context.ref.replace('refs/tags/v', ''); + const branch = `release-${version}`; + const base = 'main'; // или другая основная ветка + + const prs = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: `${context.repo.owner}:${branch}`, + base + }); + + if (prs.data.length === 0) { + await github.rest.pulls.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head: branch, + base: base, + title: `Release ${version}`, + body: `This PR prepares the release \`${version}\`.`, + draft: true + }); + console.log(`Created draft pull request from ${branch} to ${base}`); + } else { + console.log(`Pull request already exists from ${branch} to ${base}`); + } + + - name: Create or reuse draft release + if: steps.check_release.outputs.skip == 'false' + id: create_release + uses: actions/github-script@v7 + with: + script: | + const tag = context.ref.replace('refs/tags/', ''); + const releases = await github.rest.repos.listReleases({ + owner: context.repo.owner, + repo: context.repo.repo + }); + + let release = releases.data.find(r => r.tag_name === tag); + if (!release) { + release = await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: tag, + name: `Release ${tag}`, + draft: true, + prerelease: false + }); + } + core.setOutput('upload_url', release.upload_url); + + - name: Upload assets + if: steps.check_release.outputs.skip == 'false' + run: make upload_assets + + - name: Delete pushed tag + if: steps.check_release.outputs.skip == 'false' + run: | + git push --delete origin ${GITHUB_REF#refs/tags/} diff --git a/Makefile b/Makefile index 706b9d43..6ea3ffd5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,13 @@ .PHONY: manifests repos assets -build: +build-deps: + @command -V find docker skopeo jq gh helm > /dev/null + @yq --version | grep -q "mikefarah" || (echo "mikefarah/yq is required" && exit 1) + @tar --version | grep -q GNU || (echo "GNU tar is required" && exit 1) + @sed --version | grep -q GNU || (echo "GNU sed is required" && exit 1) + @awk --version | grep -q GNU || (echo "GNU awk is required" && exit 1) + +build: build-deps make -C packages/apps/http-cache image make -C packages/apps/postgres image make -C packages/apps/mysql image @@ -38,10 +45,9 @@ assets: make -C packages/core/installer/ assets test: - test -f _out/assets/nocloud-amd64.raw.xz || make -C packages/core/installer talos-nocloud make -C packages/core/testing apply make -C packages/core/testing test - make -C packages/core/testing test-applications + #make -C packages/core/testing test-applications generate: hack/update-codegen.sh diff --git a/hack/e2e.sh b/hack/e2e.sh index c99b7cc1..0ab859f0 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -84,7 +84,7 @@ done # Start VMs for i in 1 2 3; do - qemu-system-x86_64 -machine type=pc,accel=kvm -cpu host -smp 4 -m 8192 \ + qemu-system-x86_64 -machine type=pc,accel=kvm -cpu host -smp 8 -m 16384 \ -device virtio-net,netdev=net0,mac=52:54:00:12:34:5$i -netdev tap,id=net0,ifname=cozy-srv$i,script=no,downscript=no \ -drive file=srv$i/system.img,if=virtio,format=raw \ -drive file=srv$i/seed.img,if=virtio,format=raw \ diff --git a/packages/core/builder/Chart.yaml b/packages/core/builder/Chart.yaml deleted file mode 100755 index 91337a4b..00000000 --- a/packages/core/builder/Chart.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: v2 -name: builder -version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/core/builder/Makefile b/packages/core/builder/Makefile deleted file mode 100755 index 9eee5594..00000000 --- a/packages/core/builder/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -NAMESPACE=cozy-builder -NAME := builder - -TALOS_VERSION=$(shell awk '/^version:/ {print $$2}' ../installer/images/talos/profiles/installer.yaml) - -include ../../../scripts/common-envs.mk - -help: ## Show this help. - @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) - -show: - helm template -n $(NAMESPACE) $(NAME) . - -apply: ## Create builder sandbox in existing Kubernetes cluster. - helm template -n $(NAMESPACE) $(NAME) . | kubectl apply -f - - docker buildx ls | grep -q '^buildkit-builder*' || docker buildx create \ - --bootstrap \ - --name=buildkit-$(NAME) \ - --driver=kubernetes \ - --driver-opt=namespace=$(NAMESPACE),replicas=1 \ - --platform=linux/amd64 \ - --platform=linux/arm64 \ - --use \ - --config config.toml - -diff: - helm template -n $(NAMESPACE) $(NAME) . | kubectl diff -f - - -delete: ## Remove builder sandbox from existing Kubernetes cluster. - kubectl delete deploy -n $(NAMESPACE) $(NAME)-talos-imager - docker buildx rm buildkit-$(NAME) - -wait-for-builder: - kubectl wait deploy --for=condition=Progressing -n $(NAMESPACE) $(NAME)-talos-imager - kubectl wait pod --for=condition=Ready -n $(NAMESPACE) -l app=$(NAME)-talos-imager diff --git a/packages/core/builder/config.toml b/packages/core/builder/config.toml deleted file mode 100644 index a3cb03a3..00000000 --- a/packages/core/builder/config.toml +++ /dev/null @@ -1,11 +0,0 @@ -[worker.oci] - gc = true - gckeepstorage = 50000 - - [[worker.oci.gcpolicy]] - keepBytes = 10737418240 - keepDuration = 604800 - filters = [ "type==source.local", "type==exec.cachemount", "type==source.git.checkout"] - [[worker.oci.gcpolicy]] - all = true - keepBytes = 53687091200 diff --git a/packages/core/builder/templates/sandbox.yaml b/packages/core/builder/templates/sandbox.yaml deleted file mode 100755 index 0a850cb6..00000000 --- a/packages/core/builder/templates/sandbox.yaml +++ /dev/null @@ -1,43 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Release.Namespace }} - labels: - pod-security.kubernetes.io/enforce: privileged ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Release.Name }}-talos-imager - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: {{ .Release.Name }}-talos-imager - strategy: - type: Recreate - template: - metadata: - labels: - app: {{ .Release.Name }}-talos-imager - spec: - automountServiceAccountToken: false - terminationGracePeriodSeconds: 1 - containers: - - name: imager - image: "{{ .Values.talos.imager.image }}" - securityContext: - privileged: true - command: - - sleep - - infinity - volumeMounts: - - mountPath: /dev - name: dev - volumes: - - hostPath: - path: /dev - type: Directory - name: dev diff --git a/packages/core/builder/values.yaml b/packages/core/builder/values.yaml deleted file mode 100755 index 17150475..00000000 --- a/packages/core/builder/values.yaml +++ /dev/null @@ -1,3 +0,0 @@ -talos: - imager: - image: ghcr.io/siderolabs/imager:v1.9.5 diff --git a/packages/core/installer/Makefile b/packages/core/installer/Makefile index e29cbd16..3d5e4d2e 100644 --- a/packages/core/installer/Makefile +++ b/packages/core/installer/Makefile @@ -19,12 +19,10 @@ diff: update: hack/gen-profiles.sh - IMAGE=$$(yq '.input.baseInstaller.imageRef | sub("/installer:", "/imager:")' images/talos/profiles/installer.yaml) \ - yq -i '.talos.imager.image = strenv(IMAGE)' ../builder/values.yaml image: pre-checks image-matchbox image-cozystack image-talos -image-cozystack: run-builder +image-cozystack: make -C ../../.. repos docker buildx build -f images/cozystack/Dockerfile ../../.. \ --provenance false \ @@ -40,11 +38,11 @@ image-cozystack: run-builder yq -i '.cozystack.image = strenv(IMAGE)' values.yaml rm -f images/installer.json -image-talos: run-builder +image-talos: test -f ../../../_out/assets/installer-amd64.tar || make talos-installer skopeo copy docker-archive:../../../_out/assets/installer-amd64.tar docker://$(REGISTRY)/talos:$(call settag,$(TALOS_VERSION)) -image-matchbox: run-builder +image-matchbox: test -f ../../../_out/assets/kernel-amd64 || make talos-kernel test -f ../../../_out/assets/initramfs-metal-amd64.xz || make talos-initramfs docker buildx build -f images/matchbox/Dockerfile ../../.. \ @@ -66,10 +64,5 @@ assets: talos-iso talos-nocloud talos-metal talos-initramfs talos-kernel talos-installer talos-iso talos-nocloud talos-metal: mkdir -p ../../../_out/assets cat images/talos/profiles/$(subst talos-,,$@).yaml | \ - kubectl exec -i -n cozy-builder deploy/builder-talos-imager -- imager - - kubectl exec -n cozy-builder deploy/builder-talos-imager -- tar -C /out -czf- . | \ - tar -C ../../../_out/assets -xzf- - kubectl exec -n cozy-builder deploy/builder-talos-imager -- rm -rf /out - -run-builder: - make -C ../builder/ apply wait-for-builder + docker run --rm -i -v /dev:/dev --privileged "ghcr.io/siderolabs/imager:$(TALOS_VERSION)" --tar-to-stdout - | \ + tar -C ../../../_out/assets -xzf- diff --git a/packages/core/testing/Makefile b/packages/core/testing/Makefile index 861f9a96..ed943a95 100755 --- a/packages/core/testing/Makefile +++ b/packages/core/testing/Makefile @@ -2,6 +2,9 @@ NAMESPACE=cozy-e2e-tests NAME := sandbox CLEAN := 1 TESTING_APPS := $(shell find ../../apps -maxdepth 1 -mindepth 1 -type d | awk -F/ '{print $$NF}') +SANDBOX_NAME := cozy-e2e-sandbox + +ROOT_DIR = $(dir $(abspath $(firstword $(MAKEFILE_LIST))/../../..)) include ../../../scripts/common-envs.mk @@ -24,7 +27,6 @@ image-e2e-sandbox: --provenance false \ --tag $(REGISTRY)/e2e-sandbox:$(call settag,$(TAG)) \ --cache-from type=registry,ref=$(REGISTRY)/e2e-sandbox:latest \ - --platform linux/amd64,linux/arm64 \ --cache-to type=inline \ --metadata-file images/e2e-sandbox.json \ --push=$(PUSH) \ @@ -34,27 +36,20 @@ image-e2e-sandbox: yq -i '.e2e.image = strenv(IMAGE)' values.yaml rm -f images/e2e-sandbox.json -copy-hack-dir: - tar -C ../../../ -cf- hack | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- tar -xf- +test: ## Run the end-to-end tests in existing sandbox. + docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && export COZYSTACK_INSTALLER_YAML=$$(helm template -n cozy-system installer ./packages/core/installer) && hack/e2e.sh' -copy-image: - cat ../../../_out/assets/nocloud-amd64.raw.xz | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -xec 'xz --decompress > /nocloud-amd64.raw' - -test: wait-for-sandbox copy-hack-dir copy-image ## Run the end-to-end tests in existing sandbox. - helm template -n cozy-system installer ../installer | kubectl exec -i -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'cat > /cozystack-installer.yaml' - kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'export COZYSTACK_INSTALLER_YAML=$$(cat /cozystack-installer.yaml) && /hack/e2e.sh' - -test-applications: wait-for-sandbox copy-hack-dir ## Run the end-to-end tests in existing sandbox for applications. +test-applications: ## Run the end-to-end tests in existing sandbox for applications. for app in $(TESTING_APPS); do \ - kubectl exec -ti -n cozy-e2e-tests deploy/cozystack-e2e-sandbox -- bash -c "/hack/e2e.application.sh $${app}"; \ + docker exec ${SANDBOX_NAME} bash -c "/hack/e2e.application.sh $${app}"; \ done - kubectl exec -ti -n cozy-e2e-tests deploy/cozystack-e2e-sandbox -- bash -c "kubectl get hr -A | grep -v 'True'" + docker exec ${SANDBOX_NAME} bash -c "kubectl get hr -A | grep -v 'True'" delete: ## Remove sandbox from existing Kubernetes cluster. - kubectl delete deploy -n $(NAMESPACE) cozystack-e2e-$(NAME) + docker rm -f "${SANDBOX_NAME}" || true exec: ## Opens an interactive shell in the sandbox container. - kubectl exec -ti -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- bash + docker exec -ti "${SANDBOX_NAME}" -- bash proxy: sync-hosts ## Enable a SOCKS5 proxy server; mirrord and gost must be installed. mirrord exec --target deploy/cozystack-e2e-sandbox --target-namespace cozy-e2e-tests -- gost -L=127.0.0.1:10080 @@ -65,6 +60,6 @@ login: ## Downloads the kubeconfig into a temporary directory and runs a shell w sync-hosts: kubectl exec -n $(NAMESPACE) deploy/cozystack-e2e-$(NAME) -- sh -c 'kubectl get ing -A -o go-template='\''{{ "127.0.0.1 localhost\n"}}{{ range .items }}{{ range .status.loadBalancer.ingress }}{{ .ip }}{{ end }} {{ range .spec.rules }}{{ .host }}{{ end }}{{ "\n" }}{{ end }}'\'' > /etc/hosts' -wait-for-sandbox: - kubectl wait deploy --for=condition=Progressing -n $(NAMESPACE) cozystack-e2e-$(NAME) - kubectl wait pod --for=condition=Ready -n $(NAMESPACE) -l app=cozystack-e2e-$(NAME) +apply: delete + docker run -d --rm --name "${SANDBOX_NAME}" --privileged "$$(yq .e2e.image values.yaml)" sleep infinity + docker cp "${ROOT_DIR}" "${SANDBOX_NAME}":/workspace diff --git a/packages/core/testing/images/e2e-sandbox/Dockerfile b/packages/core/testing/images/e2e-sandbox/Dockerfile index 74e6f0cf..74eb68a2 100755 --- a/packages/core/testing/images/e2e-sandbox/Dockerfile +++ b/packages/core/testing/images/e2e-sandbox/Dockerfile @@ -1,11 +1,11 @@ FROM ubuntu:22.04 ARG KUBECTL_VERSION=1.32.0 -ARG TALOSCTL_VERSION=1.8.4 +ARG TALOSCTL_VERSION=1.9.5 ARG HELM_VERSION=3.16.4 RUN apt-get update -RUN apt-get -y install genisoimage qemu-kvm qemu-utils iproute2 iptables wget xz-utils netcat curl jq +RUN apt-get -y install genisoimage qemu-kvm qemu-utils iproute2 iptables wget xz-utils netcat curl jq make git RUN curl -LO "https://github.com/siderolabs/talos/releases/download/v${TALOSCTL_VERSION}/talosctl-linux-amd64" \ && chmod +x talosctl-linux-amd64 \ && mv talosctl-linux-amd64 /usr/local/bin/talosctl diff --git a/packages/core/testing/templates/sandbox.yaml b/packages/core/testing/templates/sandbox.yaml deleted file mode 100755 index dcb73205..00000000 --- a/packages/core/testing/templates/sandbox.yaml +++ /dev/null @@ -1,40 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Release.Namespace }} - labels: - pod-security.kubernetes.io/enforce: privileged ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cozystack-e2e-{{ .Release.Name }} - namespace: cozy-e2e-tests -spec: - replicas: 1 - selector: - matchLabels: - app: cozystack-e2e-{{ .Release.Name }} - strategy: - type: Recreate - template: - metadata: - labels: - app: cozystack-e2e-{{ .Release.Name }} - spec: - automountServiceAccountToken: false - terminationGracePeriodSeconds: 1 - containers: - - name: sandbox - image: "{{ .Values.e2e.image }}" - securityContext: - privileged: true - env: - - name: KUBECONFIG - value: /kubeconfig - - name: TALOSCONFIG - value: /talosconfig - command: - - sleep - - infinity diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index 104eec02..04ffe5f9 100755 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v0.29.0@sha256:e24763eba3831224c6f92a09063c4dd199a15a4a8bc68171451e4c5106b9f5f3 + image: ghcr.io/cozystack/cozystack/e2e-sandbox:latest@sha256:f41b1e0f76e7820ce87cd3dfa1b680490d0c898105690c79537943f26ab28040