From bdc3f83f8976d10c2fa4010f6fc2e1a4e6871fb1 Mon Sep 17 00:00:00 2001 From: Max Brenner Date: Wed, 23 Jun 2021 16:50:30 +0200 Subject: [PATCH] add testbed matrix Signed-off-by: Max Brenner --- .github/workflows/{nightly.yml => sanity.yml} | 79 ++++++++++++++----- 1 file changed, 58 insertions(+), 21 deletions(-) rename .github/workflows/{nightly.yml => sanity.yml} (79%) diff --git a/.github/workflows/nightly.yml b/.github/workflows/sanity.yml similarity index 79% rename from .github/workflows/nightly.yml rename to .github/workflows/sanity.yml index 1f3d87775..3efd37ba9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/sanity.yml @@ -1,4 +1,4 @@ -name: nightly build +name: sanity testing env: # thirdparties DOCKER_SERVER: tip-tip-wlan-cloud-docker-repo.jfrog.io @@ -19,9 +19,9 @@ env: on: workflow_dispatch: inputs: - testbed: - default: 'basic-02' - description: 'Testbed to test' + testbeds: + default: '["basic-01","basic-02"]' + description: 'Testbed(s) to test' required: false marker_expression: default: 'sanity' @@ -117,9 +117,29 @@ jobs: # kubectl get pods -n tip # kubectl describe pods -n tip + generate-matrix: + name: generate testbed matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: generate-matrix + id: set-matrix + run: | + set -x + TESTBEDS="${{ github.event.inputs.testbeds || '[\"basic-01\",\"basic-02\"]' }}" + echo "$TESTBEDS" + TESTBED_ARRAY=$(echo "$TESTBEDS" | jq -c 'map({"testbed":.})') + echo "$TESTBED_ARRAY" + echo "::set-output name=matrix::{\"include\":${TESTBED_ARRAY}}" + test: runs-on: ubuntu-latest - needs: [ build ] + needs: [ build, generate-matrix ] + strategy: + max-parallel: 1 + fail-fast: false + matrix: ${{ fromJson( needs.generate-matrix.outputs.matrix ) }} steps: - name: get EKS access credentials run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} @@ -131,7 +151,15 @@ jobs: - name: set job name id: job - run: echo "::set-output name=name::nightly-ci-${{ github.run_number }}" + run: echo "::set-output name=name::testing-${{ github.run_number }}" + + - name: prepare namespace + id: namespace + run: | + NAMESPACE="testing-${{ github.run_number }}-${{ matrix.testbed }}" + kubectl create ns $NAMESPACE + kubectl config set-context --current --namespace=$NAMESPACE + echo "::set-output name=name::${NAMESPACE}" - name: create configuration.py secret run: | @@ -160,7 +188,7 @@ jobs: - -c - | cd tests - pytest -m "${{ github.event.inputs.marker_expression || 'sanity' }}" -s -vvv --testbed="${{ github.event.inputs.testbed || 'basic-02' }}" --skip-testrail --alluredir=/tmp/allure-results + pytest -m "${{ github.event.inputs.marker_expression || 'sanity' }}" -s -vvv --testbed="${{ matrix.testbed }}" --skip-testrail --alluredir=/tmp/allure-results ret=\$? # sleep some time to be able to download the Allure results sleep 60 @@ -180,10 +208,15 @@ jobs: backoffLimit: 0 EOF - sleep 60 # wait for the pod to come up + # wait for pod to spawn + sleep 1 podname=$(kubectl get pods --no-headers -o custom-columns=":metadata.name" -l job-name="${{ steps.job.outputs.name }}" | sed "s/pod\///") + kubectl wait "pod/$podname" --for condition=ready + + #sleep 30 # wait for the pod to come up + until [ -s test_everything.xml ] do sleep 10 @@ -210,19 +243,22 @@ jobs: if: always() uses: actions/upload-artifact@v2 with: - name: allure-results + name: allure-results-${{ matrix.testbed }} path: allure-results - name: cleanup if: always() run: | - kubectl delete job "${{ steps.job.outputs.name }}" --wait=true --ignore-not-found=true - kubectl delete secret configuration --wait=true --ignore-not-found=true + kubectl delete ns "${{ steps.namespace.outputs.name }}" --wait=true report: - needs: [ test ] - if: always() runs-on: ubuntu-latest + needs: [ test, generate-matrix ] + if: always() + strategy: + max-parallel: 1 + fail-fast: false + matrix: ${{ fromJson( needs.generate-matrix.outputs.matrix ) }} steps: - name: install Allure CLI tool run: | @@ -231,7 +267,7 @@ jobs: - uses: actions/download-artifact@v2 with: - name: allure-results + name: allure-results-${{ matrix.testbed }} path: allure-results - name: checkout testing repo @@ -248,14 +284,14 @@ jobs: - name: copy history into results run: | - if [ -e "reports/sanity/latest" ] ; then - cp -r reports/sanity/latest/history/ allure-results/history + if [ -e "reports/sanity/${{ matrix.testbed }}/latest" ] ; then + cp -r reports/sanity/${{ matrix.testbed }}/latest/history/ allure-results/history fi - name: add report metadata run: | cat << EOF >> allure-results/environment.properties - Testbed=${{ github.event.inputs.testbed || 'basic-02' }} + Testbed=${{ matrix.testbed }} Tests.CommitId=$(cd wlan-testing && git rev-parse --short HEAD) CiRun.Id=${{ github.run_id }} CiRun.Number=${{ github.run_number }} @@ -268,18 +304,18 @@ jobs: - name: upload Allure report as artifact uses: actions/upload-artifact@v2 with: - name: allure-report + name: allure-report-${{ matrix.testbed }} path: allure-report - name: copy new report if: ${{ (github.event.inputs.marker_expression || 'sanity') == 'sanity' }} run: | - mkdir -p reports/sanity - cp -Tr allure-report reports/sanity/${{ github.run_number }} + mkdir -p reports/sanity/${{ matrix.testbed }} + cp -Tr allure-report reports/sanity/${{ matrix.testbed }}/${{ github.run_number }} - name: update latest symlink if: ${{ (github.event.inputs.marker_expression || 'sanity') == 'sanity' }} - working-directory: reports/sanity + working-directory: reports/sanity/${{ matrix.testbed }} run: ln -fns ${{ github.run_number }} latest - name: generate new index.html @@ -295,6 +331,7 @@ jobs: git commit -m "Automated deployment: $(date -u)" - name: push + if: github.ref == 'refs/heads/master' uses: ad-m/github-push-action@v0.6.0 with: github_token: ${{ secrets.GITHUB_TOKEN }}