diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml new file mode 100644 index 000000000..b1e28f0d1 --- /dev/null +++ b/.github/actions/run-tests/action.yml @@ -0,0 +1,136 @@ +name: run tests on Kubernetes +description: create Kubernetes job that executes pytest code + +inputs: + namespace: + description: a name for the Kubernetes namespace that will be used + required: true + testbed: + description: the testbed string that will be passed to the --testbed parameter + required: true + marker_expression: + description: the marker expression that will be passed to the -m parameter + required: true + configuration: + description: the configuration string that will be used to create the configuration secret + required: true + testing_docker_image: + description: Docker image to use when executing tests + required: true + additional_args: + description: additional arguments that will be passed to the pytest execution string + required: false + allure_results_artifact_name: + description: name of the artifact that the allure results will be uploaded to + required: false + default: allure-results + +runs: + using: "composite" + steps: + - name: create and select namespace + shell: bash + run: | + kubectl create ns ${{ inputs.namespace }} + kubectl config set-context --current --namespace=${{ inputs.namespace }} + + - name: set job name + shell: bash + id: job + run: echo "::set-output name=name::testing" + + - name: create configuration.py secret + shell: bash + run: | + cat << EOF > configuration.py + ${{ inputs.configuration }} + EOF + kubectl create secret generic configuration --from-file=configuration=./configuration.py + + - name: run tests + shell: bash + run: | + cat </dev/null 2>&1 + done + echo "tests completed" + echo "downloading allure results..." + kubectl cp $podname:/tmp/allure-results allure-results >/dev/null 2>&1 + echo "waiting for pod to exit" + kubectl logs -f $podname >/dev/null 2>&1 + exit $(kubectl get pod $podname --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}") + + - name: upload Allure results as artifact + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.allure_results_artifact_name }} + path: allure-results + + - name: cleanup + if: ${{ always() }} + shell: bash + run: | + kubectl delete ns "${{ inputs.namespace }}" --wait=true diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index c8a038e38..6f6a981bc 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -1,10 +1,6 @@ name: interop testing env: - # thirdparties - DOCKER_SERVER: tip-tip-wlan-cloud-docker-repo.jfrog.io - DOCKER_USER_NAME: wlan-testing-cicd - DOCKER_USER_PASSWORD: ${{ secrets.DOCKER_USER_PASSWORD }} # AWS credentials AWS_EKS_NAME: tip-wlan-main AWS_DEFAULT_OUTPUT: json @@ -19,11 +15,6 @@ env: on: workflow_dispatch: - inputs: - additional_markers: - default: '' - description: 'Pass additional markers that will be and-combined with the interop marker, e.g. "twog or fiveg" -> "interop and twog or fiveg"' - required: false schedule: - cron: '30 20 * * *' @@ -35,172 +26,203 @@ jobs: build: runs-on: ubuntu-latest steps: - # checkout needed repositories - - name: Checkout Testing repo - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + - name: build and push Docker image + uses: ./.github/actions/build-and-push-docker with: - path: wlan-testing + registry: tip-tip-wlan-cloud-docker-repo.jfrog.io + registry_user: wlan-testing-cicd + registry_password: ${{ secrets.DOCKER_USER_PASSWORD }} - - name: Checkout LANforge scripts - uses: actions/checkout@v2 - with: - path: wlan-lanforge-scripts - repository: Telecominfraproject/wlan-lanforge-scripts - - - name: import LANforge scripts - working-directory: wlan-testing - run: ./sync_repos.bash - - # build and push docker image - - name: docker login - run: docker login ${{ env.DOCKER_SERVER }} -u ${{ env.DOCKER_USER_NAME }} -p ${{ env.DOCKER_USER_PASSWORD }} - - name: build docker image - working-directory: wlan-testing - run: docker build -t ${{ env.DOCKER_SERVER }}/cloud-sdk-nightly:${{ github.run_id }} -f docker/Dockerfile . - - name: push docker image - run: docker push ${{ env.DOCKER_SERVER }}/cloud-sdk-nightly:${{ github.run_id }} - - - test: + # interop-01 + test-galaxy-s9: runs-on: ubuntu-latest needs: [ build ] - strategy: - fail-fast: false - max-parallel: 1 - matrix: - test_type: [android, ios] - - outputs: - additional_markers: ${{ steps.marker.outputs.additional }} - + continue-on-error: true steps: + - uses: actions/checkout@v2 + - name: get EKS access credentials run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} - - name: install Allure CLI tool - run: | - wget https://repo.maven.apache.org/maven2/io/qameta/allure/allure-commandline/${{ env.ALLURE_CLI_VERSION }}/allure-commandline-${{ env.ALLURE_CLI_VERSION }}.tgz - tar -xzf allure-commandline-${{ env.ALLURE_CLI_VERSION }}.tgz - - - name: set job name - id: job - run: echo "::set-output name=name::interop-ci-${{ github.run_number }}-${{ matrix.test_type }}" - - - name: create configuration.py secret - run: | - cat << EOF > configuration.py - ${{ secrets.LAB_CONFIGURATION }} - EOF - kubectl create secret generic configuration --from-file=configuration=./configuration.py - - - name: calculate marker expression - id: marker - run: | - if [ "${{ matrix.test_type }}" = "android" ]; then - MARKER_EXPRESSION="interop_and and interop_uc_sanity and client_connect" - else - MARKER_EXPRESSION="interop_ios and interop_uc_sanity and client_connect" - fi - - ADDITIONAL_MARKERS="${{ github.event.inputs.additional_markers || '' }}" - if [ ! -z "$ADDITIONAL_MARKERS" ]; then - MARKER_EXPRESSION="$MARKER_EXPRESSION and ${ADDITIONAL_MARKERS}" - fi - - echo "::set-output name=additional::${ADDITIONAL_MARKERS}" - echo "::set-output name=expression::${MARKER_EXPRESSION}" - - - name: run interop tests - run: | - cat </dev/null 2>&1 - done - echo "tests completed" - echo "downloading allure results..." - kubectl cp $podname:/tmp/allure-results allure-results >/dev/null 2>&1 - echo "waiting for pod to exit" - kubectl logs -f $podname >/dev/null 2>&1 - exit $(kubectl get pod $podname --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}") - - - name: upload Allure results as artifact - if: ${{ always() }} - uses: actions/upload-artifact@v2 + - name: run tests + uses: ./.github/actions/run-tests with: - name: allure-results-${{ matrix.test_type }} - path: allure-results + namespace: interop-${{ github.run_id }}-galaxy-s9 + testbed: interop-01 + marker_expression: "interop_uc_sanity and client_connect and android" + configuration: "${{ secrets.LAB_CONFIGURATION }}" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o model-android='Galaxy S9' -o 'jobName=Github-Interop-galaxy-s9' -o 'jobNumber=${{ github.run_number }}' --skip-lanforge" + allure_results_artifact_name: allure-results-galaxy-s9 - - name: cleanup - if: ${{ always() }} - run: | - kubectl delete job "${{ steps.job.outputs.name }}" --wait=true --ignore-not-found=true - kubectl delete secret configuration --wait=true --ignore-not-found=true + test-iphone-12: + runs-on: ubuntu-latest + needs: [ test-galaxy-s9 ] + continue-on-error: true + steps: + - uses: actions/checkout@v2 + + - name: get EKS access credentials + run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} + + - name: run tests + uses: ./.github/actions/run-tests + with: + namespace: interop-${{ github.run_id }}-iphone-12 + testbed: interop-01 + marker_expression: "interop_uc_sanity and client_connect and ios" + configuration: "${{ secrets.LAB_CONFIGURATION }}" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o model-iOS='iPhone-12' -o 'jobName=Github-Interop-iphone-12' -o 'jobNumber=${{ github.run_number }}' --skip-lanforge" + allure_results_artifact_name: allure-results-iphone-12 + + test-pixel-4: + runs-on: ubuntu-latest + needs: [ test-iphone-12 ] + continue-on-error: true + steps: + - uses: actions/checkout@v2 + + - name: get EKS access credentials + run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} + + - name: run tests + uses: ./.github/actions/run-tests + with: + namespace: interop-${{ github.run_id }}-pixel-4 + testbed: interop-01 + marker_expression: "interop_uc_sanity and client_connect and android" + configuration: "${{ secrets.LAB_CONFIGURATION }}" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o model-android='Pixel 4' -o 'jobName=Github-Interop-pixel-4' -o 'jobNumber=${{ github.run_number }}' --skip-lanforge" + allure_results_artifact_name: allure-results-pixel-4 + + # interop-02 + test-galaxy-s10: + runs-on: ubuntu-latest + needs: [ build ] + continue-on-error: true + steps: + - uses: actions/checkout@v2 + + - name: get EKS access credentials + run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} + + - name: run tests + uses: ./.github/actions/run-tests + with: + namespace: interop-${{ github.run_id }}-galaxy-s10 + testbed: interop-02 + marker_expression: "interop_uc_sanity and client_connect and android" + configuration: "${{ secrets.LAB_CONFIGURATION }}" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o model-android='Galaxy S10.*' -o 'jobName=Github-Interop-galaxy-s10' -o 'jobNumber=${{ github.run_number }}' --skip-lanforge" + allure_results_artifact_name: allure-results-galaxy-s10 + + test-iphone-7: + runs-on: ubuntu-latest + needs: [ test-galaxy-s10 ] + continue-on-error: true + steps: + - uses: actions/checkout@v2 + + - name: get EKS access credentials + run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} + + - name: run tests + uses: ./.github/actions/run-tests + with: + namespace: interop-${{ github.run_id }}-iphone-7 + testbed: interop-02 + marker_expression: "interop_uc_sanity and client_connect and ios" + configuration: "${{ secrets.LAB_CONFIGURATION }}" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o model-iOS='iPhone-7' -o 'jobName=Github-Interop-iphone-7' -o 'jobNumber=${{ github.run_number }}' --skip-lanforge" + allure_results_artifact_name: allure-results-iphone-7 + + test-iphone-11: + runs-on: ubuntu-latest + needs: [ test-iphone-7 ] + continue-on-error: true + steps: + - uses: actions/checkout@v2 + + - name: get EKS access credentials + run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} + + - name: run tests + uses: ./.github/actions/run-tests + with: + namespace: interop-${{ github.run_id }}-iphone-11 + testbed: interop-02 + marker_expression: "interop_uc_sanity and client_connect and ios" + configuration: "${{ secrets.LAB_CONFIGURATION }}" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o model-iOS='iPhone-11' -o 'jobName=Github-Interop-iphone-11' -o 'jobNumber=${{ github.run_number }}' --skip-lanforge" + allure_results_artifact_name: allure-results-iphone-11 + + # interop-03 + test-galaxy-s20: + runs-on: ubuntu-latest + needs: [ build ] + continue-on-error: true + steps: + - uses: actions/checkout@v2 + + - name: get EKS access credentials + run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} + + - name: run tests + uses: ./.github/actions/run-tests + with: + namespace: interop-${{ github.run_id }}-galaxy-s20 + testbed: interop-03 + marker_expression: "interop_uc_sanity and client_connect and android" + configuration: "${{ secrets.LAB_CONFIGURATION }}" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o model-android='Galaxy S20' -o 'jobName=Github-Interop-galaxy-s20' -o 'jobNumber=${{ github.run_number }}' --skip-lanforge" + allure_results_artifact_name: allure-results-galaxy-s20 + + test-iphone-xr: + runs-on: ubuntu-latest + needs: [ test-galaxy-s20 ] + continue-on-error: true + steps: + - uses: actions/checkout@v2 + + - name: get EKS access credentials + run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} + + - name: run tests + uses: ./.github/actions/run-tests + with: + namespace: interop-${{ github.run_id }}-iphone-xr + testbed: interop-03 + marker_expression: "interop_uc_sanity and client_connect and ios" + configuration: "${{ secrets.LAB_CONFIGURATION }}" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o model-iOS='iPhone-XR' -o 'jobName=Github-Interop-iphone-xr' -o 'jobNumber=${{ github.run_number }}' --skip-lanforge" + allure_results_artifact_name: allure-results-iphone-xr report: - needs: [ test ] + needs: [ test-iphone-11, test-iphone-xr, test-pixel-4 ] if: always() runs-on: ubuntu-latest + strategy: + max-parallel: 1 + fail-fast: false + matrix: + device: + - galaxy-s9 + - galaxy-s10 + - galaxy-s20 + - pixel-4 + - iphone-7 + - iphone-xr + - iphone-11 + - iphone-12 + steps: - name: install Allure CLI tool run: | @@ -209,13 +231,8 @@ jobs: - uses: actions/download-artifact@v2 with: - name: allure-results-android - path: allure-results-android - - - uses: actions/download-artifact@v2 - with: - name: allure-results-ios - path: allure-results-ios + name: allure-results-${{ matrix.device }} + path: allure-results - name: checkout testing repo uses: actions/checkout@v2 @@ -231,14 +248,14 @@ jobs: - name: copy history into results run: | - if [ -e "reports/interop/interop/latest" ] ; then - cp -r reports/interop/interop/latest/history/ allure-results-ios/history - cp -r reports/interop/interop/latest/history/ allure-results-android/history + if [ -e "reports/interop/${{ matrix.device }}/latest" ] ; then + cp -r reports/interop/${{ matrix.device }}/latest/history/ allure-results/history fi - name: add report metadata run: | - cat << EOF >> allure-results-android/environment.properties + cat << EOF >> allure-results/environment.properties + Testbed=interop Tests.CommitId=$(cd wlan-testing && git rev-parse --short HEAD) CiRun.Id=${{ github.run_id }} @@ -247,28 +264,27 @@ jobs: EOF - name: generate Allure report - run: allure-${{ env.ALLURE_CLI_VERSION }}/bin/allure generate allure-results-android allure-results-ios + run: allure-${{ env.ALLURE_CLI_VERSION }}/bin/allure generate allure-results - name: upload Allure report as artifact uses: actions/upload-artifact@v2 with: - name: allure-report + name: allure-report-${{ matrix.device }} path: allure-report # doing this to be able to aggregate multiple reports together later on - name: copy results into report run: | mkdir -p allure-report/results - cp -r allure-results-android/* allure-report/results - cp -r allure-results-ios/* allure-report/results + cp -r allure-results/* allure-report/results - name: copy new report run: | - mkdir -p reports/interop/interop - cp -Tr allure-report reports/interop/interop/${{ github.run_number }} + mkdir -p reports/interop/${{ matrix.device }} + cp -Tr allure-report reports/interop/${{ matrix.device }}/${{ github.run_number }} - name: update latest symlink - working-directory: reports/interop/interop + working-directory: reports/interop/${{ matrix.device }} run: ln -fns ${{ github.run_number }} latest - name: generate new index.html @@ -296,15 +312,19 @@ jobs: uses: ./wlan-testing/.github/actions/allure-report-to-s3 with: test_type: interop - testbed: interop + testbed: ${{ matrix.device }} report_path: allure-report s3_access_key_id: ${{ secrets.ALLURE_S3_ACCESS_KEY_ID }} s3_access_key_secret: ${{ secrets.ALLURE_S3_ACCESS_KEY_SECRET }} cleanup: - needs: [ test ] + needs: [ test-iphone-11, test-iphone-xr, test-pixel-4 ] runs-on: ubuntu-latest if: always() steps: + - uses: actions/checkout@v2 - name: cleanup Docker image - run: curl -u${{ env.DOCKER_USER_NAME }}:${{ env.DOCKER_USER_PASSWORD }} -X DELETE "https://tip.jfrog.io/artifactory/tip-wlan-cloud-docker-repo/cloud-sdk-nightly/${{ github.run_id }}" + uses: ./.github/actions/cleanup-docker + with: + registry_user: wlan-testing-cicd + registry_password: ${{ secrets.DOCKER_USER_PASSWORD }} diff --git a/.quali/get_configuration.py b/.quali/get_configuration.py index 61557937f..26d1d773a 100644 --- a/.quali/get_configuration.py +++ b/.quali/get_configuration.py @@ -5,6 +5,7 @@ from cloudshell.api.cloudshell_api import UpdateTopologyGlobalInputsRequest, Upd from common import get_session + def get_attribute_value(cloudshell_session, attribute): if attribute.Type == 'Boolean': return True if attribute.Value == 'True' else False @@ -26,7 +27,7 @@ def main(): config = { 'controller': {}, - 'access_point':[], + 'access_point': [], 'traffic_generator': {} } @@ -35,8 +36,9 @@ def main(): continue config['controller']['url'] = f'https://sec-{res_id.split("-")[0]}.cicd.lab.wlan.tip.build:16001' - config['controller']['username'] = next(attr.Value for attr in service.Attributes if attr.Name == f'{service.ServiceName}.User') - #config['controller']['password'] = next(attr.Value for attr in service.Attributes if attr.Name == f'{service.ServiceName}.Password') + config['controller']['username'] = next( + attr.Value for attr in service.Attributes if attr.Name == f'{service.ServiceName}.User') + # config['controller']['password'] = next(attr.Value for attr in service.Attributes if attr.Name == f'{service.ServiceName}.Password') config['controller']['password'] = 'OpenWifi%123' for resource in resources_in_reservation: @@ -67,8 +69,8 @@ def main(): 'ip': tf_config['ip'], 'port': tf_config['port'], 'ssh_port': tf_config['ssh_port'], - '2.4G-Radio': [tf_config['lf_2dot4G_Radio']], - '5G-Radio': [tf_config['lf_5G_Radio']], + '2.4G-Radio': tf_config['lf_2dot4G_Radio'].replace(' ', '').split(','), + '5G-Radio': tf_config['lf_5G_Radio'].replace(' ', '').split(','), 'AX-Radio': tf_config['AX_Radio'].replace(' ', '').split(','), 'upstream': tf_config['Upstream'], 'upstream_subnet': tf_config['upstream_subnet'], @@ -83,5 +85,6 @@ def main(): print(repr(config)) + if __name__ == '__main__': main() diff --git a/libs/perfecto_libs/iOS_lib.py b/libs/perfecto_libs/iOS_lib.py index c6435fc04..0c0dac087 100644 --- a/libs/perfecto_libs/iOS_lib.py +++ b/libs/perfecto_libs/iOS_lib.py @@ -920,12 +920,11 @@ def get_ip_address_ios(request, WifiName, WifiPass, setup_perfectoMobile, connDa openApp(connData["bundleId-iOS-Settings"], setup_perfectoMobile) try: - time.sleep(2) - driver.implicitly_wait(2) + time.sleep(1) try: print("Verifying Connected Wifi Connection") report.step_start("Loading Wifi Page") - element = driver.find_element_by_xpath("//XCUIElementTypeCell[@name='Wi-Fi']") + element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((MobileBy.XPATH, "//XCUIElementTypeCell[@name='Wi-Fi']"))) element.click() except NoSuchElementException: print("Exception: Verify Xpath - unable to click on Wifi") @@ -948,7 +947,7 @@ def get_ip_address_ios(request, WifiName, WifiPass, setup_perfectoMobile, connDa get_wifi_switch_element = driver.find_element_by_xpath("//*[@label='Wi-Fi' and @value='1']") get_wifi_switch_element_text = get_wifi_switch_element.text except: - print("switch is OFF") + print("Switch is OFF") if get_wifi_switch_element_text == "1" or get_wifi_switch_element_text == 1: print("WIFI Switch is ON") @@ -983,17 +982,16 @@ def get_ip_address_ios(request, WifiName, WifiPass, setup_perfectoMobile, connDa try: print("getting in to Additional details") - additional_details_element = driver.find_element_by_xpath( - "//*[@label='selected']/parent::*/parent::*/XCUIElementTypeButton[@label='More Info']") + additional_details_element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((MobileBy.XPATH, "//*[@label='selected']/parent::*/parent::*/XCUIElementTypeButton[@label='More Info']"))) additional_details_element.click() try: print("Forget Connected Network") - forget_ssid = driver.find_element_by_xpath("//*[@label='Forget This Network']") + forget_ssid = WebDriverWait(driver, 10).until(EC.presence_of_element_located((MobileBy.XPATH, "//*[@label='Forget This Network']"))) forget_ssid.click() print("Forget old ssid") try: report.step_start("Forget SSID popup1") - forget_ssid_popup = driver.find_element_by_xpath("//*[@label='Forget']") + forget_ssid_popup = WebDriverWait(driver, 10).until(EC.presence_of_element_located((MobileBy.XPATH, "//*[@label='Forget']"))) forget_ssid_popup.click() print("**alert** Forget SSID popup killed **alert**") @@ -1053,7 +1051,6 @@ def get_ip_address_ios(request, WifiName, WifiPass, setup_perfectoMobile, connDa # ---------------------Set Password------------------------------- try: - driver.implicitly_wait(5) wifiPassword = driver.find_element_by_xpath("//*[@label='Password']") wifiPassword.send_keys(WifiPass) except NoSuchElementException: @@ -1062,8 +1059,7 @@ def get_ip_address_ios(request, WifiName, WifiPass, setup_perfectoMobile, connDa # ---------------------Click on join------------------------------- try: - driver.implicitly_wait(5) - joinBTN = driver.find_element_by_xpath("//*[@label='Join']") + joinBTN = WebDriverWait(driver, 10).until(EC.presence_of_element_located((MobileBy.XPATH, "//*[@label='Join']"))) joinBTN.click() except Exception as e: print("Join Button Not Enabled...Password may not be needed") @@ -1103,9 +1099,8 @@ def get_ip_address_ios(request, WifiName, WifiPass, setup_perfectoMobile, connDa try: time.sleep(2) - driver.implicitly_wait(2) report.step_start("Forget Network") - forget_ssid = driver.find_element_by_xpath("//*[@label='Forget This Network']") + forget_ssid = WebDriverWait(driver, 10).until(EC.presence_of_element_located((MobileBy.XPATH, "//*[@label='Forget This Network']"))) forget_ssid.click() print("Forget old ssid") # time.sleep(2)