wlan-testing/.github/workflows/regression.yml

name: regression testing

env:
  # AWS credentials
  AWS_EKS_NAME: tip-wlan-main
  AWS_DEFAULT_OUTPUT: json
  AWS_DEFAULT_REGION: ap-south-1
  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CLIENT_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CLIENT_KEY }}
  # Cloud SDK certs
  CACERT: ${{ secrets.CACERT }}
  CAKEY: ${{ secrets.CAKEY }}
  ALLURE_CLI_VERSION: 2.25.0

on:
  workflow_dispatch:
    inputs:
      openwifi_revision:
        required: true
        default: "main"
        description: "revision of the Open Wifi Helm chart"
      ap_models:
        required: true
        default: "edgecore_oap103,hfcl_ion4xe,edgecore_eap104"
        description: "the AP models to test"
      ap_version:
        required: true
        default: "next-latest"
        description: "revision of firmware to flash on AP, <branch>-<commit>"
      marker_expressions:
        required: true
        default: "schema_validation_tests"
        description: "Marker expressions to select tests to execute"
      additional_args:
        default: ""
        description: "additional arguments that will be passed to the pytest execution"
        required: false
      existing_controller:
        required: true
        default: "qa01"
        description: "Use an existing cloud controller, e.g. like qa01 instead of dynamic one"
      tests_release:
        required: false
        default: "WIFI-15250"
        description: "Tests release branch to use (i.e. 'release/v2.8.0' or 'master'). If left empty, latest release branch is used"

jobs:
  # Set vars
  vars:
    runs-on: ubuntu-latest
    outputs:
      openwifi_revision: ${{ steps.vars.outputs.openwifi}}
      ap_models: ${{ steps.vars.outputs.ap_models}}
      ap_version: ${{ steps.vars.outputs.ap_version}}
      marker_expressions: ${{ steps.vars.outputs.marker_expressions }}
      additional_arguments: ${{ steps.vars.outputs.additional_arguments }}
      existing_controller: ${{ steps.vars.outputs.existing_controller }}
      tests_release: ${{ steps.vars.outputs.tests_release }}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: set variables
        id: vars
        run: |
          echo "openwifi=$(echo ${{ github.event.inputs.openwifi_revision || 'main' }})" >> $GITHUB_OUTPUT
          echo "ap_version=$(echo ${{ github.event.inputs.ap_version || 'next-latest' }})" >> $GITHUB_OUTPUT
          echo "existing_controller=$(echo ${{ github.event.inputs.existing_controller || 'qa01' }})" >> $GITHUB_OUTPUT

          # Use the provided AP models or default to our test APs
          AP_MODELS="${{ github.event.inputs.ap_models || 'edgecore_oap103,hfcl_ion4xe,edgecore_eap104' }}"
          MARKER_EXPRESSIONS="${{ github.event.inputs.marker_expressions || 'schema_validation_tests' }}"
          
          echo "additional_arguments=$(echo ${{ github.event.inputs.additional_args || ''}})" >> $GITHUB_OUTPUT
          
          MARKER_EXPRESSIONS=$(echo $MARKER_EXPRESSIONS | sed "s/,/\",\"/g" | sed 's/^/[\"/g' | sed 's/$/\"]/g')
          AP_MODELS=$(echo $AP_MODELS | sed "s/,/\",\"/g" | sed 's/^/[\"/g' | sed 's/$/\"]/g')
          
          cat >> $GITHUB_OUTPUT << EOF
          marker_expressions=${MARKER_EXPRESSIONS}
          ap_models=${AP_MODELS}
          EOF
          
          # Use WIFI-15250 branch for testing
          echo "tests_release=WIFI-15250" >> $GITHUB_OUTPUT

  # Build test image
  build:
    runs-on: ubuntu-latest
    needs: ["vars"]
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ needs.vars.outputs.tests_release }}
      - name: build and push Docker image
        uses: ./.github/actions/build-and-push-docker
        with:
          registry: tip-tip-wlan-cloud-docker-repo.jfrog.io
          registry_user: ${{ secrets.DOCKER_USER_NAME }}
          registry_password: ${{ secrets.DOCKER_USER_PASSWORD }}

  # Run tests on edgecore_oap103
  test-edgecore-oap103:
    needs: ["vars", "build"]
    runs-on: [self-hosted, small]
    timeout-minutes: 120
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'edgecore_oap103')"
    env:
      AP_MODEL: edgecore_oap103
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      # TODO WIFI-7839 delete when issue is resolved on AWS CLI side
      - name: install kubectl
        run: |
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: get EKS access credentials
        run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="regression-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run schema_validation tests
        uses: ./.github/actions/run-tests
        if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')"
        with:
          namespace: ${{ steps.namespace.outputs.name }}-schema-validation
          testbed: basic-3
          marker_expression: "ow_regression_lf and schema_validation_tests and through_github"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }}"
          allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_through_github
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"


      - name: Check if Allure results exist
        if: always()
        id: check-allure
        run: |
          if [ -d "allure-results" ]; then
            echo "allure-results-exist=true" >> $GITHUB_OUTPUT
          else
            echo "allure-results-exist=false" >> $GITHUB_OUTPUT
          fi

      - name: Upload Allure results as artifact
        if: always() && steps.check-allure.outputs.allure-results-exist == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-${{ env.AP_MODEL }}-${{ github.run_id }}
          path: allure-results/
          retention-days: 1

      - name: Generate immediate test status
        if: always()
        run: |
          echo "## Test Status for ${{ env.AP_MODEL }}" >> $GITHUB_STEP_SUMMARY
          if [ "${{ job.status }}" == "success" ]; then
            echo "✅ **PASSED** - All tests completed successfully" >> $GITHUB_STEP_SUMMARY
          else
            echo "❌ **FAILED** - Some tests failed" >> $GITHUB_STEP_SUMMARY
          fi

          echo "📊 **Report available at:** http://openwifi-allure-reports.s3-website-us-east-1.amazonaws.com/regression/schema_validation_tests/${{ env.AP_MODEL }}/${{ github.run_id }}/" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation

  # Run tests on hfcl_ion4xe
  test-hfcl-ion4xe:
    needs: ["vars", "build"]
    runs-on: [self-hosted, small]
    timeout-minutes: 120
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'hfcl_ion4xe')"
    env:
      AP_MODEL: hfcl_ion4xe
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      # TODO WIFI-7839 delete when issue is resolved on AWS CLI side
      - name: install kubectl
        run: |
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: get EKS access credentials
        run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="regression-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run schema_validation tests
        uses: ./.github/actions/run-tests
        if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')"
        with:
          namespace: ${{ steps.namespace.outputs.name }}-schema-validation
          testbed: basic-3a
          marker_expression: "ow_regression_lf and schema_validation_tests"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }}"
          allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_tests
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      - name: Check if Allure results exist
        if: always()
        id: check-allure
        run: |
          if [ -d "allure-results" ]; then
            echo "allure-results-exist=true" >> $GITHUB_OUTPUT
          else
            echo "allure-results-exist=false" >> $GITHUB_OUTPUT
          fi

      - name: Upload Allure results as artifact
        if: always() && steps.check-allure.outputs.allure-results-exist == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-${{ env.AP_MODEL }}-${{ github.run_id }}
          path: allure-results/
          retention-days: 1

      - name: Generate immediate test status
        if: always()
        run: |
          echo "## Test Status for ${{ env.AP_MODEL }}" >> $GITHUB_STEP_SUMMARY
          if [ "${{ job.status }}" == "success" ]; then
            echo "✅ **PASSED** - All tests completed successfully" >> $GITHUB_STEP_SUMMARY
          else
            echo "❌ **FAILED** - Some tests failed" >> $GITHUB_STEP_SUMMARY
          fi

          echo "📊 **Report available at:** http://openwifi-allure-reports.s3-website-us-east-1.amazonaws.com/regression/schema_validation_tests/${{ env.AP_MODEL }}/${{ github.run_id }}/" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation

  # Run tests on edgecore_eap104
  test-edgecore-eap104:
    needs: [ "vars", "build" ]
    runs-on: [ self-hosted, small ]
    timeout-minutes: 120
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'edgecore_eap104')"
    env:
      AP_MODEL: edgecore_eap104
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      # TODO WIFI-7839 delete when issue is resolved on AWS CLI side
      - name: install kubectl
        run: |
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: get EKS access credentials
        run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="regression-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run schema_validation tests
        uses: ./.github/actions/run-tests
        if: "!cancelled() && contains(fromJSON(needs.vars.outputs.marker_expressions), 'schema_validation_tests')"
        with:
          namespace: ${{ steps.namespace.outputs.name }}-schema-validation
          testbed: basic-3
          marker_expression: "ow_regression_lf and schema_validation_tests and through_github"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: "-o firmware=${{ needs.vars.outputs.ap_version }} ${{ needs.vars.outputs.additional_arguments }}"
          allure_results_artifact_name: allure-results-${{ steps.ap_model.outputs.model }}-schema_validation_through_github
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"


      - name: Check if Allure results exist
        if: always()
        id: check-allure
        run: |
          if [ -d "allure-results" ]; then
            echo "allure-results-exist=true" >> $GITHUB_OUTPUT
          else
            echo "allure-results-exist=false" >> $GITHUB_OUTPUT
          fi

      - name: Upload Allure results as artifact
        if: always() && steps.check-allure.outputs.allure-results-exist == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-${{ env.AP_MODEL }}-${{ github.run_id }}
          path: allure-results/
          retention-days: 1

      - name: Generate immediate test status
        if: always()
        run: |
          echo "## Test Status for ${{ env.AP_MODEL }}" >> $GITHUB_STEP_SUMMARY
          if [ "${{ job.status }}" == "success" ]; then
            echo "✅ **PASSED** - All tests completed successfully" >> $GITHUB_STEP_SUMMARY
          else
            echo "❌ **FAILED** - Some tests failed" >> $GITHUB_STEP_SUMMARY
          fi

          echo "📊 **Report available at:** http://openwifi-allure-reports.s3-website-us-east-1.amazonaws.com/regression/schema_validation_tests/${{ env.AP_MODEL }}/${{ github.run_id }}/" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }}-schema-validation --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }}-schema-validation $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}-schema-validation