name: advanced testing

env:
  AWS_EKS_NAME: tip-wlan-main
  AWS_DEFAULT_OUTPUT: json
  AWS_DEFAULT_REGION: ap-south-1
  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CLIENT_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CLIENT_KEY }}
  # Cloud SDK certs
  CACERT: ${{ secrets.CACERT }}
  CAKEY: ${{ secrets.CAKEY }}
  ALLURE_CLI_VERSION: 2.25.0

on:
  workflow_dispatch:
    inputs:
      openwifi_revision:
        required: true
        default: 'main'
        description: 'revision of the Open Wifi Helm chart'
      ap_models:
        required: true
        default: 'edgecore_eap104,edgecore_eap101,edgecore_eap111,hfcl_ion4xe,cig_wf189,cig_wf189h,edgecore_eap105,sonicfi_rap7110c-341x,cig_wf196,cig_wf186h,sonicfi_rap630c-311g,yuncore_fap655,udaya_a6-id2,edgecore_eap112,edgecore_oap103'
        description: 'the AP models to test'
      ap_version:
        required: true
        default: 'next-latest'
        description: 'revision of firmware to flash on AP, <branch>-<commit>'
      marker_expression:
        required: true
        default: 'advance'
        description: 'Marker expression to select tests to execute'
      tests_release:
        required: false
        default: ""
        description: "Tests release branch to use (i.e. 'release/v2.8.0' or 'master'). If left empty, latest release branch is used"

  # schedule:
  #   - cron: '30 20 * * *'

defaults:
  run:
    shell: bash

jobs:
  vars:
    runs-on: ubuntu-latest
    outputs:
      openwifi_revision: ${{ steps.vars.outputs.openwifi }}
      ap_models: ${{ steps.vars.outputs.ap_models }}
      ap_version: ${{ steps.vars.outputs.ap_version }}
      marker_expression: ${{ steps.vars.outputs.marker_expression }}
      existing_controller: ${{ steps.vars.outputs.existing_controller }}
      tests_release: ${{ steps.vars.outputs.tests_release }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: set variables
        id: vars
        run: |
          echo "openwifi=$(echo ${{ github.event.inputs.openwifi_revision || 'main' }})" >> $GITHUB_OUTPUT
          echo "ap_models=$(echo ${{ github.event.inputs.ap_models || 'edgecore_eap104,edgecore_eap101,cig_wf196,yuncore_fap655,edgecore_oap103' }})" >> $GITHUB_OUTPUT
          echo "ap_version=$(echo ${{ github.event.inputs.ap_version || 'next-latest' }})" >> $GITHUB_OUTPUT
          echo "marker_expression=$(echo ${{ github.event.inputs.marker_expression || 'advance' }})" >> $GITHUB_OUTPUT
          echo "existing_controller=$(echo ${{ github.event.inputs.existing_controller || 'qa01' }})" >> $GITHUB_OUTPUT
          DOW=$(date +%u)
          SLOT=$(( 10#$DOW % 7 % 2 )) # 0 - slot 1, 1 - slot 2
          echo "Current date is $(date --iso-8601=s)"
          echo "Current date of week is $DOW which is $(( 10#$DOW % 7 )) in cron notation"
          echo "Current slot is $SLOT"
          if [[ "$SLOT" -eq "0" ]]; then
            echo "Choosing first slot since $SLOT == 0"
            AP_MODELS="${{ github.event.inputs.ap_models || 'edgecore_eap104,edgecore_eap101,cig_wf196,yuncore_fap655,edgecore_oap103' }}"
          else
            echo "Choosing second slot since $SLOT != 0"
            AP_MODELS="${{ github.event.inputs.ap_models || 'edgecore_oap101-6e,edgecore_eap102,cig_wf186h' }}"
          fi
          echo "List of AP models to test - $AP_MODELS"
          AP_MODELS=$(echo $AP_MODELS | sed "s/,/\",\"/g" | sed 's/^/[\"/g' | sed 's/$/\"]/g')
          cat >> $GITHUB_OUTPUT << EOF
          ap_models=${AP_MODELS}
          EOF
          LATEST_TESTS_RELEASE=$(git branch -r | grep 'release/v' | sed 's!\s*origin/!!' | tail -1)
          if [[ -z "${{ github.event.inputs.tests_release }}" ]]; then
            echo "Tests release was not passed, using branch $LATEST_TESTS_RELEASE"
            echo "tests_release=$LATEST_TESTS_RELEASE" >> $GITHUB_OUTPUT
          else
            echo "Tests release was passed - ${{ github.event.inputs.tests_release }}"
            echo "tests_release=${{ github.event.inputs.tests_release }}" >> $GITHUB_OUTPUT
          fi
          cat $GITHUB_OUTPUT

  build:
    runs-on: ubuntu-latest
    needs: ["vars"]
    steps:
    - uses: actions/checkout@v3
      with:
        ref: ${{ needs.vars.outputs.tests_release }}
    - name: build and push Docker image
      uses: ./.github/actions/build-and-push-docker
      with:
        registry: tip-tip-wlan-cloud-docker-repo.jfrog.io
        registry_user: ${{ secrets.DOCKER_USER_NAME }}
        registry_password: ${{ secrets.DOCKER_USER_PASSWORD }}

  test-edgecore-eap104:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'edgecore_eap104')"
    env:
      AP_MODEL: edgecore_eap104
    steps:
    - name: Set AP model output
      id: ap_model
      run: |
        echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.8"

    - uses: actions/checkout@v3
    - name: install aws CLI tool
      run: |
        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
        unzip awscliv2.zip
        sudo ./aws/install

    - name: install kubectl
      run: |
        # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
        curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
        sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

    - name: get EKS access credentials
      run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

    - name: prepare namespace name
      id: namespace
      run: |
        NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
        echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

    - name: prepare configuration
      run: |
        cat << EOF > lab_info.json
        ${{ secrets.LAB_INFO_JSON }}
        EOF

    - name: run tests
      uses: ./.github/actions/run-tests
      with:
        namespace: ${{ steps.namespace.outputs.name }}
        testbed: advanced-1e
        marker_expression: "${{ needs.vars.outputs.marker_expression }}"
        configuration_file: "./lab_info.json"
        testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
        additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
        allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
        dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

    # necessary because if conditionals in composite actions are currently not respected
    - name: get tests logs
      if: always()
      continue-on-error: true
      run: |
        podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
        kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

    - name: delete namespace
      if: always()
      continue-on-error: true
      run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

    - name: show gw logs
      if: failure()
      run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

    - name: show fms logs
      if: failure()
      run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

    - name: show prov logs
      if: failure()
      run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

    - name: show sec logs
      if: failure()
      run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

    - name: show analytics logs
      if: failure()
      run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

    - name: show subscription (userportal) logs
      if: failure()
      run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-edgecore-eap101:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'edgecore_eap101')"
    env:
      AP_MODEL: edgecore_eap101
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
         python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-1b
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-edgecore-eap111:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'edgecore_eap111')"
    env:
      AP_MODEL: edgecore_eap111
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-1c
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-edgecore-eap112:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'edgecore_eap112')"
    env:
      AP_MODEL: edgecore_eap112
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-1a
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-hfcl-ion4xe:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'hfcl_ion4xe')"
    env:
      AP_MODEL: hfcl_ion4xe
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-1d
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-cig-wf189:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'cig_wf189')"
    env:
      AP_MODEL: cig_wf189
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
         python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-2a
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-edgecore-eap105:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'edgecore_eap105')"
    env:
      AP_MODEL: edgecore_eap105
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
         python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-2b
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-sonicfi-rap7110c-341x:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'sonicfi_rap7110c-341x')"
    env:
      AP_MODEL: sonicfi_rap7110c-341x
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-2c
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-cig-wf189h:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'cig_wf189h')"
    env:
      AP_MODEL: cig_wf189h
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-2d
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-cig-wf186h:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'cig_wf186h')"
    env:
      AP_MODEL: cig_wf186h
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
         python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-3a
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-cig-wf196:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'cig_wf196')"
    env:
      AP_MODEL: cig_wf196
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-1e
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-sonicfi-rap630c-311g:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'sonicfi_rap630c-311g')"
    env:
      AP_MODEL: sonicfi_rap630c-311g
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-1h
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-udaya-a6-id2:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'udaya_a6-id2')"
    env:
      AP_MODEL: udaya_a6-id2
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-3d
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-yuncore-fap655:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'yuncore_fap655')"
    env:
      AP_MODEL: yuncore_fap655
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-1f
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  test-edgecore-oap103:
    runs-on: [ self-hosted, small ]
    needs: [ "vars", "build" ]
    timeout-minutes: 1440
    if: "!cancelled() && contains(fromJSON(needs.vars.outputs.ap_models), 'edgecore_oap103')"
    env:
      AP_MODEL: edgecore_oap103
    steps:
      - name: Set AP model output
        id: ap_model
        run: |
          echo "model=${AP_MODEL}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - uses: actions/checkout@v3
      - name: install aws CLI tool
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install

      - name: install kubectl
        run: |
          # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side
          curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl"
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

      - name: get EKS access credentials
        run: aws eks update-kubeconfig  --name ${{ env.AWS_EKS_NAME }}

      - name: prepare namespace name
        id: namespace
        run: |
          NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ steps.ap_model.outputs.model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
          echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT

      - name: prepare configuration
        run: |
          cat << EOF > lab_info.json
          ${{ secrets.LAB_INFO_JSON }}
          EOF

      - name: run tests
        uses: ./.github/actions/run-tests
        with:
          namespace: ${{ steps.namespace.outputs.name }}
          testbed: advanced-1g
          marker_expression: "${{ needs.vars.outputs.marker_expression }}"
          configuration_file: "./lab_info.json"
          testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
          additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"'
          allure_results_artifact_name: "allure-results-${{ steps.ap_model.outputs.model }}"
          dns_records_to_resolve: "sec-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build gw-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build fms-${{ needs.vars.outputs.existing_controller }}.cicd.lab.wlan.tip.build"

      # necessary because if conditionals in composite actions are currently not respected
      - name: get tests logs
        if: always()
        continue-on-error: true
        run: |
          podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
          kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true

      - name: delete namespace
        if: always()
        continue-on-error: true
        run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }}

      - name: show gw logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owgw

      - name: show fms logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owfms

      - name: show prov logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owprov

      - name: show sec logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsec

      - name: show analytics logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/analytics

      - name: show subscription (userportal) logs
        if: failure()
        run: kubectl -n openwifi-${{ needs.vars.outputs.existing_controller }} logs deployment/owsub

  # report
  report:
    if: "!cancelled()"
    runs-on: ubuntu-latest
    needs: [ vars, test-edgecore-eap104, test-edgecore-eap101, test-edgecore-eap111, test-hfcl-ion4xe, test-cig-wf189, test-cig-wf189h, test-edgecore-eap105, test-sonicfi-rap7110c-341x, test-cig-wf196, test-cig-wf186h, test-sonicfi-rap630c-311g, test-yuncore-fap655, test-udaya-a6-id2, test-edgecore-eap112, test-edgecore-oap103 ]
    strategy:
      fail-fast: false
      matrix:
        model: ${{ fromJson( needs.vars.outputs.ap_models ) }}

    steps:
    - name: checkout testing repo
      uses: actions/checkout@v3

    - uses: actions/download-artifact@v4
      with:
        name: allure-results-${{ matrix.model }}
        path: allure-results

    - name: download history of previous run
      continue-on-error: true
      run: |
        LAST_RUN_ID=$(aws s3api head-object --bucket openwifi-allure-reports --key advanced/${{ matrix.model }}/latest/index.html | jq -r .Metadata.latest)
        aws s3 cp --recursive s3://openwifi-allure-reports/advanced/${{ matrix.model }}/$LAST_RUN_ID/history history

    - name: generate Allure report
      uses: ./.github/actions/generate-allure-report
      with:
        results_path: ./allure-results
        history_path: ./history
        additional_metadata: |
          Ap.Model=${{ matrix.model }}

    - name: upload Allure report as artifact
      uses: actions/upload-artifact@v4
      with:
        name: allure-report-${{ matrix.model }}
        path: allure-report

    # doing this to be able to aggregate multiple reports together later on
    - name: copy results into report
      run: |
        cp -r allure-results allure-report/results

    - name: upload to S3
      # if: github.ref == 'refs/heads/master' && needs.vars.outputs.marker_expression == 'advance'
      uses: ./.github/actions/allure-report-to-s3
      with:
        test_type: advanced
        testbed: ${{ matrix.model }}
        report_path: allure-report
        s3_access_key_id: ${{ secrets.ALLURE_S3_ACCESS_KEY_ID }}
        s3_access_key_secret: ${{ secrets.ALLURE_S3_ACCESS_KEY_SECRET }}

  cleanup:
    needs: [ test-edgecore-eap104, test-edgecore-eap101, test-edgecore-eap111, test-hfcl-ion4xe, test-cig-wf189, test-cig-wf189h, test-edgecore-eap105, test-sonicfi-rap7110c-341x, test-cig-wf196, test-cig-wf186h, test-sonicfi-rap630c-311g, test-yuncore-fap655, test-udaya-a6-id2, test-edgecore-eap112, test-edgecore-oap103 ]
    runs-on: ubuntu-latest
    if: always()
    steps:
    - uses: actions/checkout@v3
    - name: cleanup Docker image
      uses: ./.github/actions/cleanup-docker
      with:
        registry_user: ${{ secrets.DOCKER_USER_NAME }}
        registry_password: ${{ secrets.DOCKER_USER_PASSWORD }}
