From b70f2cc7d17ccc309c96b5fdfd70738521de5b89 Mon Sep 17 00:00:00 2001 From: bhargavi-ct Date: Tue, 16 Sep 2025 11:53:43 +0530 Subject: [PATCH] Wifi 14790 (#1064) * Add multiple master configurations in schema validation tests Signed-off-by: bhargavi-ct * Update schema_validation script for proper visualization of discrepancies Signed-off-by: bhargavi-ct * update test_state_pretty_through_github testcase for better reporting Signed-off-by: bhargavi-ct * Add Schema Validation tests in Regression Suite Signed-off-by: bhargavi-ct * Add schema_validation_tests marker to default marker in regression workflow Signed-off-by: bhargavi-ct * Add ow_regression_lf marker to schema_validation_tests Signed-off-by: bhargavi-ct * Add --commit-id argument to additional_args in regression workflow Signed-off-by: bhargavi-ct * Add commit-id argument to additional_args in regression workflow Signed-off-by: bhargavi-ct * Add additional_arguments in vars Signed-off-by: bhargavi-ct * Update regression workflow to take additional_args through github events Signed-off-by: bhargavi-ct * Update regression workflow to take additional_args through github events for schema_validation tests Signed-off-by: bhargavi-ct --------- Signed-off-by: bhargavi-ct Co-authored-by: bhargavi-ct --- .github/workflows/regression.yml | 394 +++++++++++++++++- .../test_schema_validation.py | 53 ++- 2 files changed, 426 insertions(+), 21 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 44096289c..703bcdcbd 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -30,7 +30,7 @@ on: description: "revision of firmware to flash on AP, -" marker_expressions: required: true - default: "dfs_tests,multi_psk_tests,rate_limiting_tests,rate_limiting_with_radius_tests,dynamic_vlan_tests,multi_vlan_tests,strict_forwarding_tests,advanced_captive_portal_tests,firmware_upgrade_downgrade,asb_tests" + default: "dfs_tests,multi_psk_tests,rate_limiting_tests,rate_limiting_with_radius_tests,dynamic_vlan_tests,multi_vlan_tests,strict_forwarding_tests,advanced_captive_portal_tests,firmware_upgrade_downgrade,asb_tests,schema_validation_tests" description: "Marker expressions to select tests to execute" additional_args: default: "" @@ -72,7 +72,7 @@ jobs: echo "ap_version=$(echo ${{ github.event.inputs.ap_version || 'next-latest' }})" >> $GITHUB_OUTPUT echo "existing_controller=$(echo ${{ github.event.inputs.existing_controller || 'qa01' }})" >> $GITHUB_OUTPUT - MARKER_EXPRESSIONS="${{ github.event.inputs.marker_expressions || 'dfs_tests,multi_psk_tests,rate_limiting_tests,rate_limiting_with_radius_tests,dynamic_vlan_tests,multi_vlan_tests,strict_forwarding_tests,advanced_captive_portal_tests,firmware_upgrade_downgrade,asb_tests' }}" + MARKER_EXPRESSIONS="${{ github.event.inputs.marker_expressions || 'dfs_tests,multi_psk_tests,rate_limiting_tests,rate_limiting_with_radius_tests,dynamic_vlan_tests,multi_vlan_tests,strict_forwarding_tests,advanced_captive_portal_tests,firmware_upgrade_downgrade,asb_tests,schema_validation_tests' }}" MARKER_EXPRESSIONS=$(echo $MARKER_EXPRESSIONS | sed "s/,/\",\"/g" | sed 's/^/[\"/g' | sed 's/$/\"]/g') echo "additional_arguments=$(echo ${{ github.event.inputs.additional_arguments || ''}})" >> $GITHUB_OUTPUT cat >> $GITHUB_OUTPUT << EOF @@ -203,6 +203,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-3a + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -535,6 +561,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-1 + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -868,6 +920,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-6 + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -1201,6 +1279,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-2 + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -1534,6 +1638,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-5a + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -1866,6 +1996,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-1a + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -2199,6 +2355,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-2a + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -2531,6 +2713,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-4 + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -2858,6 +3066,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-6a + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -3190,6 +3424,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-3 + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -3522,6 +3782,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-5 + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -3854,6 +4140,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-4a + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -4186,6 +4498,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-3 + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -4518,6 +4856,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-1 + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" @@ -4850,6 +5214,32 @@ jobs: continue-on-error: true run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-dfs + - name: run schema_validation tests + uses: ./.github/actions/run-tests + if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')" + with: + namespace: ${{ steps.namespace.outputs.name }}-schema-validation + testbed: basic-4 + marker_expression: "ow_regression_lf and schema_validation_tests" + configuration_file: "./lab_info.json" + testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} + additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }} ${{ github.event.inputs.additional_args }}" + allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests + dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build" + + # necessary because if conditionals in composite actions are currently not respected + - name: get tests logs + if: always() + continue-on-error: true + run: | + podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") + kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true + + - name: delete namespace + if: always() + continue-on-error: true + run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation + - name: run multipsk tests uses: ./.github/actions/run-tests if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'multi_psk_tests')" diff --git a/tests/e2e/basic/schema_validation/test_schema_validation.py b/tests/e2e/basic/schema_validation/test_schema_validation.py index 88b63ba57..acf8c5c56 100644 --- a/tests/e2e/basic/schema_validation/test_schema_validation.py +++ b/tests/e2e/basic/schema_validation/test_schema_validation.py @@ -9,7 +9,7 @@ import time from tabulate import tabulate from datetime import datetime -pytestmark = [pytest.mark.schema_validation] +pytestmark = [pytest.mark.schema_validation_tests, pytest.mark.ow_regression_lf] # Get the directory of the current test config file @@ -22,12 +22,12 @@ with open(file_path, 'r') as file: config_data_1 = json.loads(json_string) file_path2 = os.path.join(test_file_dir, 'master-config-2.json') -with open(file_path, 'r') as file: +with open(file_path2, 'r') as file: json_string = file.read() config_data_2 = json.loads(json_string) file_path3 = os.path.join(test_file_dir, 'master-config-3.json') -with open(file_path, 'r') as file: +with open(file_path3, 'r') as file: json_string = file.read() config_data_3 = json.loads(json_string) @@ -50,7 +50,6 @@ def get_github_file(url, path=None, commit_id=None): "wlan-ucentral-schema repo.") return response.text - def validate_schema_through_github(commit_id, path): def get_commit_id(owner, repo, path="", headers=None): if headers is None: @@ -116,7 +115,8 @@ def validate_schema_through_github(commit_id, path): allure.attach(updated_schema_pretty_json, name=f"NEW {path}:") if updated_schema_pretty_json == previous_schema_pretty_json: - logging.info(f"No changes found at {path}. Exiting.") + logging.info(f"No changes were found at {path}. Exiting.") + allure.attach(name=f"No changes found at {path}", body=f"No changes were found at {path}. Exiting the test.") return None, None, None else: logging.info(f"Changes found at {path}. Proceeding with the comparison.") @@ -124,6 +124,15 @@ def validate_schema_through_github(commit_id, path): json.loads(updated_schema_pretty_json)) return added_keys, removed_keys, changed_items + def convert_changed_items_to_vertical_table(changed_items): + formatted_rows = [] + for path, old, new in changed_items: + formatted_rows.append(["Key Path", path]) + formatted_rows.append(["Old Value", old]) + formatted_rows.append(["New Value", new]) + formatted_rows.append(["", ""]) # Add blank line between entries + return tabulate(formatted_rows, headers=["Field", "Content"], tablefmt="fancy_grid") + if commit_id is None: logging.info("Use --commit-id to the pass an old commit-id of tip/wlan-ucentral-schema repo. Skipping the test.") pytest.skip("Use --commit-id to the pass an old commit-id of tip/wlan-ucentral-schema repo. Skipping the test.") @@ -164,15 +173,17 @@ def validate_schema_through_github(commit_id, path): if changed_items: changed_items = [list(key) for key in changed_items] changed_items = sorted(changed_items) - message = ("Note: The value at these key paths have been modified.\n\n" - + tabulate(changed_items, headers=['Key Paths', 'Old Value', 'New Value'], tablefmt='fancy_grid')) + message = ( + "Note: The following key paths have modified values:\n\n" + + convert_changed_items_to_vertical_table(changed_items) + ) logging.info("\nChanged Items:\n" + message + "\n") allure.attach(message, name="Changed items:") pytest.fail(f"Differences found in the schema, check Test Body for Added/Removed/Changed items") return -def validate_state_message_through_ap(test_object, target_object, config_data): +def validate_state_message_through_ap(test_object, target_object, config_data, ssid=None): def get_type_of_message(message): type_of_message = "unknown" if isinstance(message, dict): @@ -316,8 +327,9 @@ def validate_state_message_through_ap(test_object, target_object, config_data): radio_5g = dict_all_radios_5g[radio][0] break + logging.info(f"ssid:{ssid}") test_object.pre_cleanup() - fiveg_sta_got_ip = test_object.client_connect_using_radio(ssid="captive-credential-4", + fiveg_sta_got_ip = test_object.client_connect_using_radio(ssid=ssid, passkey="OpenWifi", security="wpa2", radio=radio_5g, station_name=["station-5G"], @@ -448,7 +460,7 @@ class TestSchemaValidationThroughGitHub(object): Objective is to identify any modifications, additions, or removals in the file: ucentral.schema.json. Unique Marker: - schema_validation and through_github and schema_json + schema_validation_tests and through_github and schema_json """ validate_schema_through_github(commit_id, "/ucentral.schema.json") @@ -466,7 +478,7 @@ class TestSchemaValidationThroughGitHub(object): Objective is to identify any modifications, additions, or removals in the file: ucentral.schema.full.json. Unique Marker: - schema_validation and through_github and schema_full_json + schema_validation_tests and through_github and schema_full_json """ validate_schema_through_github(commit_id, "/ucentral.schema.full.json") @@ -483,7 +495,7 @@ class TestSchemaValidationThroughGitHub(object): Objective is to identify any modifications, additions, or removals in the file: ucentral.schema.pretty.json. Unique Marker: - schema_validation and through_github and schema_pretty_json + schema_validation_tests and through_github and schema_pretty_json """ validate_schema_through_github(commit_id, "/ucentral.schema.pretty.json") @@ -500,7 +512,7 @@ class TestSchemaValidationThroughGitHub(object): Objective is to identify any modifications, additions, or removals in the file: ucentral.state.pretty.json. Unique Marker: - schema_validation and through_github and state_pretty_json + schema_validation_tests and through_github and state_pretty_json """ validate_schema_through_github(commit_id, "/ucentral.state.pretty.json") @@ -524,9 +536,10 @@ class TestSchemaValidationThroughAPTerminal(object): Objective is to detect discrepancies in data types (e.g., string to integer) and object structures. Unique Marker: - schema_validation and through_ap_terminal and master_config_1 + schema_validation_tests and through_ap_terminal and master_config_1 """ - validate_state_message_through_ap(get_test_library, get_target_object, config_data=config_data_1) + ssid = "captive-credential-4" + validate_state_message_through_ap(get_test_library, get_target_object, config_data=config_data_1, ssid=ssid) @pytest.mark.master_config_2 @allure.title("Pushing master config-2") @@ -541,9 +554,10 @@ class TestSchemaValidationThroughAPTerminal(object): Objective is to detect discrepancies in data types (e.g., string to integer) and object structures. Unique Marker: - schema_validation and through_ap_terminal and master_config_2 + schema_validation_tests and through_ap_terminal and master_config_2 """ - validate_state_message_through_ap(get_test_library, get_target_object, config_data=config_data_2) + ssid = "captive-uam-8" + validate_state_message_through_ap(get_test_library, get_target_object, config_data=config_data_2, ssid=ssid) @pytest.mark.master_config_3 @allure.title("Pushing master config-3") @@ -558,6 +572,7 @@ class TestSchemaValidationThroughAPTerminal(object): Objective is to detect discrepancies in data types (e.g., string to integer) and object structures. Unique Marker: - schema_validation and through_ap_terminal and master_config_3 + schema_validation_tests and through_ap_terminal and master_config_3 """ - validate_state_message_through_ap(get_test_library, get_target_object, config_data=config_data_3) + ssid = "Uchannel-ds-4" + validate_state_message_through_ap(get_test_library, get_target_object, config_data=config_data_3, ssid=ssid)