TOOLS-125 enable nightly pytest execution (#50)

* adjust nightly execution to pytest changes

Signed-off-by: Max Brenner <xamrennerb@gmail.com>

.github/workflows/nightly.yml

@@ -2,10 +2,8 @@ name: nightly build
 env:
   # thirdparties
   DOCKER_SERVER: tip-tip-wlan-cloud-docker-repo.jfrog.io
-  DOCKER_USER_NAME: ${{ secrets.DOCKER_USER_NAME }}
+  DOCKER_USER_NAME: wlan-testing-cicd
   DOCKER_USER_PASSWORD: ${{ secrets.DOCKER_USER_PASSWORD }}
-  TESTRAIL_USER_PASSWORD: ${{ secrets.TESTRAIL_USER_PASSWORD }}
-  JFROG_USER_PASSWORD: ${{ secrets.JFROG_USER_PASSWORD }}
   # AWS credentials
   AWS_EKS_NAME: tip-wlan-main
   AWS_DEFAULT_OUTPUT: json
@@ -16,12 +14,20 @@ env:
   # Cloud SDK certs
   CACERT: ${{ secrets.CACERT }}
   CAKEY: ${{ secrets.CAKEY }}
+  ALLURE_CLI_VERSION: 2.14.0
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - master
   schedule:
   - cron: '15 0 * * *'
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -31,117 +37,186 @@ jobs:
       uses: actions/checkout@v2
       with:
         path: wlan-testing
+
     - name: Checkout LANforge scripts
       uses: actions/checkout@v2
       with:
         path: wlan-lanforge-scripts
         repository: Telecominfraproject/wlan-lanforge-scripts
 
+    - name: import LANforge scripts
+      working-directory: wlan-testing
+      run: ./sync_repos.bash
+
     # build and push docker image
     - name: docker login
-      shell: bash
       run: docker login ${{ env.DOCKER_SERVER }} -u ${{ env.DOCKER_USER_NAME }} -p ${{ env.DOCKER_USER_PASSWORD }}
     - name: build docker image
-      shell: bash
+      working-directory: wlan-testing
-      run: docker build -t ${{ env.DOCKER_SERVER }}/cloud-sdk-nightly:pytest-${{ github.run_id }} -f wlan-testing/pytest/Dockerfile .
+      run: docker build -t ${{ env.DOCKER_SERVER }}/cloud-sdk-nightly:pytest-${{ github.run_number }} -f docker/Dockerfile .
     - name: push docker image
-      shell: bash
+      run: docker push ${{ env.DOCKER_SERVER }}/cloud-sdk-nightly:pytest-${{ github.run_number }}
-      run: docker push ${{ env.DOCKER_SERVER }}/cloud-sdk-nightly:pytest-${{ github.run_id }}
 
-  cloudsdk:
+#  cloudsdk:
-    runs-on: ubuntu-latest
+#    runs-on: ubuntu-latest
-    steps:
+#    steps:
-    - name: Checkout pki scripts repo
+#    - name: Checkout pki scripts repo
-      uses: actions/checkout@v2
+#      uses: actions/checkout@v2
-      with:
+#      with:
-        path: wlan-pki
+#        path: wlan-pki
-        repository: Telecominfraproject/wlan-pki-cert-scripts
+#        repository: Telecominfraproject/wlan-pki-cert-scripts
-    - name: Checkout Cloud SDK repo
+#    - name: Checkout Cloud SDK repo
-      uses: actions/checkout@v2
+#      uses: actions/checkout@v2
-      with:
+#      with:
-        path: wlan-helm
+#        path: wlan-helm
-        repository: Telecominfraproject/wlan-cloud-helm
+#        repository: Telecominfraproject/wlan-cloud-helm
-    - name: Checkout helm values repo
+#    - name: Checkout helm values repo
-      uses: actions/checkout@v2
+#      uses: actions/checkout@v2
-      with:
+#      with:
-        path: toolsmith
+#        path: toolsmith
-        repository: Telecominfraproject/Toolsmith
+#        repository: Telecominfraproject/Toolsmith
-        token: ${{ secrets.PAT_TOKEN }}
+#        token: ${{ secrets.PAT_TOKEN }}
-
+#
-    - name: Prepare environment
+#    - name: Prepare environment
-      shell: bash
+#      run: |
-      run: |
+#        mkdir -p ./wlan-pki/testCA/private
-        mkdir -p ./wlan-pki/testCA/private
+#        mkdir -p ./wlan-pki/testCA/newcerts
-        mkdir -p ./wlan-pki/testCA/newcerts
+#        mkdir -p ./wlan-pki/generated
-        mkdir -p ./wlan-pki/generated
+#        touch ./wlan-pki/testCA/index.txt
-        touch ./wlan-pki/testCA/index.txt
+#        echo "01" > ./wlan-pki/testCA/serial.txt
-        echo "01" > ./wlan-pki/testCA/serial.txt
+#        echo "${{ env.CACERT }}" | base64 -d > ./wlan-pki/testCA/cacert.pem
-        echo "${{ env.CACERT }}" | base64 -d > ./wlan-pki/testCA/cacert.pem
+#        echo "${{ env.CAKEY }}" | base64 -d > ./wlan-pki/testCA/private/cakey.pem
-        echo "${{ env.CAKEY }}" | base64 -d > ./wlan-pki/testCA/private/cakey.pem
+#        cp ./toolsmith/helm-values/aws-cicd.yaml ./wlan-helm/tip-wlan/resources/environments/aws-cicd.yaml
-        cp ./toolsmith/helm-values/aws-cicd.yaml ./wlan-helm/tip-wlan/resources/environments/aws-cicd.yaml
+#
-
+#    - name: Generate certs
-    - name: Generate certs
+#      run: |
-      shell: bash
+#        cd ./wlan-pki
-      run: |
+#        ./generate_all.sh true
-        cd ./wlan-pki
+#        ./copy-certs-to-helm.sh "../wlan-helm"
-        ./generate_all.sh true
+#
-        ./copy-certs-to-helm.sh "../wlan-helm"
+#    - name: Deploy Cloud SDK
-
+#      run: |
-    - name: Deploy Cloud SDK
+#        cd ./wlan-helm/tip-wlan
-      shell: bash
+#        aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}
-      run: |
+#        # start deployment
-        cd ./wlan-helm/tip-wlan
+#        helm dependency update
-        aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}
+#        helm upgrade --install tip . -f resources/environments/aws-cicd.yaml --create-namespace --namespace tip
-        # start deployment
+#
-        helm dependency update
+#    - name: Show pod state on deployment failure
-        helm upgrade --install tip . -f resources/environments/aws-cicd.yaml --create-namespace --namespace tip
+#      if: failure()
+#      run: |
+#        kubectl get pods -n tip
+#        kubectl describe pods -n tip
 
   test:
     runs-on: ubuntu-latest
-    needs: [ build, cloudsdk ]
+    needs: [ build ]
     steps:
-    - name: run tests
+    - name: get EKS access credentials
-      run: |
+      run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}
-        today=$(date +"%d-%m-%Y")
-        aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}
-        kubectl delete job nightly-ci-$today --wait=true --ignore-not-found=true
 
+    - name: install Allure CLI tool
+      run: |
+        wget https://repo.maven.apache.org/maven2/io/qameta/allure/allure-commandline/${{ env.ALLURE_CLI_VERSION }}/allure-commandline-${{ env.ALLURE_CLI_VERSION }}.tgz
+        tar -xzf allure-commandline-${{ env.ALLURE_CLI_VERSION }}.tgz
+
+    - name: set job name
+      id: job
+      run: echo "::set-output name=name::nightly-ci-${{ github.run_number }}"
+
+    - name: create configuration.py secret
+      run: |
+        cat << EOF > configuration.py
+        ${{ secrets.LAB_CONFIGURATION }}
+        EOF
+
+        kubectl create secret generic configuration --from-file=configuration=./configuration.py
+
+    - name: run sanity tests
+      run: |
         cat <<EOF | kubectl apply -f -
         apiVersion: batch/v1
         kind: Job
         metadata:
-          name: nightly-ci-$today
+          name: "${{ steps.job.outputs.name }}"
         spec:
           template:
             spec:
               containers:
               - name: tests
-                image: ${{ env.DOCKER_SERVER }}/cloud-sdk-nightly:pytest-${{ github.run_id }}
+                image: ${{ env.DOCKER_SERVER }}/cloud-sdk-nightly:pytest-${{ github.run_number }}
-                args: 
+                command:
-                - "-c"
+                  - /bin/bash
-                - "pytest --no-testrails --skip-update-firmware -o lanforge-ip-address=10.28.3.6; sleep 3000"
+                  - -x
-                command: [ "bash" ]
+                  - -c
+                  - |
+                    cd tests
+                    pytest -m sanity -s -vvv --testbed=basic-02 --skip-testrail --alluredir=/tmp/allure-results
+                    ret=\$?
+                    # sleep some time to be able to download the Allure results
+                    sleep 60
+                    exit \$ret
+                volumeMounts:
+                - name: configuration
+                  mountPath: "/wlan-testing/tests/configuration.py"
+                  subPath: configuration
+                  readOnly: true
               imagePullSecrets:
               - name: tip-docker-registry-key
               restartPolicy: Never
+              volumes:
+              - name: configuration
+                secret:
+                  secretName: configuration
           backoffLimit: 0
         EOF
-        sleep 60 # needed to wait for the pod to come up
+
-        podname=$(kubectl get pods -o name -l job-name=nightly-ci-$today | sed "s/pod\///")
+        sleep 60 # wait for the pod to come up
+
+        podname=$(kubectl get pods --no-headers -o custom-columns=":metadata.name" -l job-name="${{ steps.job.outputs.name }}" | sed "s/pod\///")
+
         until [ -s test_everything.xml ]
         do
-          echo "waiting for tests to complete"
+          sleep 10
-          kubectl cp $podname:/ci/test_everything.xml test_everything.xml
+          echo "waiting for tests to complete..."
-          sleep 15
+          kubectl cp $podname:/wlan-testing/tests/test_everything.xml test_everything.xml >/dev/null 2>&1
         done
+        echo "tests completed"
+
+        echo "downloading allure results..."
+        kubectl cp $podname:/tmp/allure-results allure-results >/dev/null 2>&1
+
+        echo "waiting for pod to exit"
+        kubectl logs -f $podname >/dev/null 2>&1
+
+        exit $(kubectl get pod $podname --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}")
+
+    - name: print logs
+      if: ${{ always() }}
+      run: |
+        podname=$(kubectl get pods --no-headers -o custom-columns=":metadata.name" -l job-name="${{ steps.job.outputs.name }}" | sed "s/pod\///")
         kubectl logs $podname
 
-        # placeholder for now
+    - name: upload Allure results as artifact
-        echo "Grafana Link: https://grafana.lab.wlan.tip.build/TO_BE_PROVIDED"
+      if: ${{ always() }}
-        echo "Kibana Link: https://kibana.lab.wlan.tip.build/TO_BE_PROVIDED"
+      uses: actions/upload-artifact@v2
-      shell: bash
-    - name: Publish Unit Test Results
-      uses: EnricoMi/publish-unit-test-result-action@v1.7
       with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
+        name: allure-results
-        files: "**/*.xml"
+        path: allure-results
+
+    - name: generate Allure report
+      if: ${{ always() }}
+      run: allure-${{ env.ALLURE_CLI_VERSION }}/bin/allure generate
+
+    - name: upload Allure report as artifact
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: allure-report
+        path: allure-report
+
+    - name: cleanup
+      if: ${{ always() }}
+      run: |
+        kubectl delete job "${{ steps.job.outputs.name }}" --wait=true --ignore-not-found=true
+        kubectl delete secret configuration --wait=true --ignore-not-found=true

.gitignore

@@ -152,3 +152,5 @@ pytest/*
 pytest/nightly*log
 
 test_everything.xml
+.vscode
+.envrc

docker/Dockerfile

@@ -1,13 +1,11 @@
 FROM python:3.8
-WORKDIR /wlan-testing
-RUN apt update && rm -rf /var/lib/apt/lists/*
-RUN pip3 install pytest==6.2.2 bs4 paramiko xlsxwriter requests pandas influxdb scp allure-pytest
 RUN mkdir ~/.pip
 RUN echo "[global]" > ~/.pip/pip.conf
 RUN echo "index-url = https://pypi.org/simple" >> ~/.pip/pip.conf
 RUN echo "extra-index-url = https://tip-read:tip-read@tip.jfrog.io/artifactory/api/pypi/tip-wlan-python-pypi-local/simple" >> ~/.pip/pip.conf
-RUN echo "tip-wlan-cloud" > ~/.pip/requirements.txt
+RUN pip3 install pytest==6.2.2 bs4 paramiko xlsxwriter requests pandas influxdb influxdb-client scp allure-pytest tip-wlan-cloud selenium perfecto-py37 Appium-Python-Client
-RUN pip3 install -r ~/.pip/requirements.txt
 COPY lanforge /wlan-testing/lanforge
 COPY tests /wlan-testing/tests
 COPY libs /wlan-testing/libs
+WORKDIR /wlan-testing
+ENTRYPOINT ["bash"]

libs/apnos/apnos.py

@@ -58,8 +58,8 @@ class APNOS:
     def ssh_cli_connect(self):
         client = paramiko.SSHClient()
         client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        print("Connecting to jumphost: %s@%s:%s with password: %s" % (
+        print("Connecting to jumphost: %s@%s:%s" % (
-            self.username, self.ip, self.port, self.password))
+            self.username, self.ip, self.port))
         client.connect(self.ip, username=self.username, password=self.password,
                        port=self.port, timeout=10, allow_agent=False, banner_timeout=200)