wlan-testing/.github/workflows/ow_docker-compose.yml

name: 2.x testing with Docker Compose deployment
env:
  # AWS credentials
  AWS_EKS_NAME: tip-wlan-main
  AWS_DEFAULT_OUTPUT: json
  AWS_DEFAULT_REGION: us-east-2
  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CLIENT_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CLIENT_KEY }}
  ALLURE_CLI_VERSION: 2.14.0

on:
  repository_dispatch:
    types: [openwifi-sdk-pr]
  workflow_dispatch:
    inputs:
      testbeds:
        default: 'basic-05'
        description: 'Testbed(s) to test'
        required: false
      marker_expression:
        default: 'sdk_restapi'
        description: 'Markers expression that will be passed to the pytest command.'
        required: false
      owgw_version:
        default: 'master'
        description: 'OpenWIFI Gateway version to be deployed.'
        required: false
      owgwui_version:
        default: 'main'
        description: 'OpenWIFI Web UI version to be deployed.'
        required: false
      owsec_version:
        default: 'main'
        description: 'OpenWIFI Security version to be deployed.'
        required: false
      owfms_version:
        default: 'main'
        description: 'OpenWIFI Firmware version to be deployed.'
        required: false
      owprov_version:
        default: 'main'
        description: 'OpenWIFI Provisioning version to be deployed.'
        required: false
      owprovui_version:
        default: 'main'
        description: 'OpenWIFI Provisioning Web UI version to be deployed.'
        required: false
      id:
        description: 'run identifier'
        required: false

defaults:
  run:
    shell: bash

jobs:
  id:
    name: Workflow ID Provider
    runs-on: ubuntu-latest
    steps:
      - name: ${{github.event.inputs.id}}
        run: echo run identifier ${{ inputs.id }}

  build:
    runs-on: ubuntu-latest
    needs: id
    steps:
    - uses: actions/checkout@v2
    - name: build and push Docker image
      uses: ./.github/actions/build-and-push-docker
      with:
        registry: tip-tip-wlan-cloud-docker-repo.jfrog.io
        registry_user: wlan-testing-cicd
        registry_password: ${{ secrets.DOCKER_USER_PASSWORD }}

  deploy-controller:
    name: Deploy OpenWIFI Cloud SDK
    runs-on: ubuntu-latest
    needs: id
    outputs:
      gateway_url: ${{ steps.gateway_url.outputs.value }}
      sec_url: ${{ steps.sec_url.outputs.value }}
      namespace: ${{ steps.namespace.outputs.value }}
    steps:
    - name: Checkout wlan-testing repo
      uses: actions/checkout@v2
      with:
        path: wlan-testing

    # build and push docker-compose-deployment image
#    - name: docker login
#      run: docker login ${{ env.DOCKER_SERVER }} -u ${{ env.DOCKER_USER_NAME }} -p ${{ env.DOCKER_USER_PASSWORD }}
#    - name: build docker image
#      working-directory: wlan-testing/docker-compose/openwifi
#      run: docker build -t ${{ env.DOCKER_SERVER }}/docker-compose-deployment:${{ github.run_id }} .
#    - name: push docker image
#      run: docker push ${{ env.DOCKER_SERVER }}/docker-compose-deployment:${{ github.run_id }}
#
    - name: Prepare certificates from secrets
      working-directory: wlan-testing/docker-compose/openwifi
      run: |
        echo "${{ secrets.DIGICERT_CERT }}" | base64 -d > websocket-cert.pem
        echo "${{ secrets.DIGICERT_KEY }}" | base64 -d > websocket-key.pem

    - name: Fetch kubeconfig
      run: |
        aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }}

    - name: Set namespace output
      id: namespace
      run: echo "::set-output name=value::ow-docker-compose-${{ github.run_id }}"

    - name: Set gateway URL output
      id: gateway_url
      run: echo "::set-output name=value::ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build"

    - name: Set sec service URL output
      id: sec_url
      run: echo "::set-output name=value::ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build"

    - name: Create Kubernetes namespace and secret
      working-directory: wlan-testing/docker-compose/openwifi
      run: |
        kubectl create ns ow-docker-compose-${{ github.run_id }}
        kubectl create secret generic tip-cicd-wildcard-certs --from-file=websocket-cert.pem=websocket-cert.pem --from-file=websocket-key.pem=websocket-key.pem -n ow-docker-compose-${{ github.run_id }}

    - name: Deploy OpenWIFI Cloud SDK
      working-directory: wlan-testing/docker-compose/openwifi
      env:
        RTTY_TOKEN: ${{ secrets.RTTY_TOKEN }}
        OWSEC_AUTHENTICATION_DEFAULT_USERNAME: ${{ secrets.UCENTRALGW_AUTH_USERNAME }}
        OWSEC_AUTHENTICATION_DEFAULT_PASSWORD: ${{ secrets.UCENTRALGW_AUTH_PASSWORD }}
        OWFMS_S3_SECRET: ${{ secrets.UCENTRALFMS_S3_SECRET }}
        OWFMS_S3_KEY: ${{ secrets.UCENTRALFMS_S3_KEY }}
      run: |
        cat <<EOF | kubectl create -f - -n ow-docker-compose-${{ github.run_id }}
        apiVersion: v1
        kind: Service
        metadata:
          name: ow-docker-compose-${{ github.run_id }}
          annotations:
            service.beta.kubernetes.io/aws-load-balancer-type: "nlb-ip"
            service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
            external-dns.alpha.kubernetes.io/hostname: ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build
            service.beta.kubernetes.io/aws-load-balancer-backend-protocol: ssl
            service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-east-2:289708231103:certificate/bfa89c7a-5b64-4a8a-bcfe-ffec655b5285"
            service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "16002,16003,443,16001,16004,16005,8443,5912,5913"
          labels:
            app: ow-docker-compose-${{ github.run_id }}
        spec:
          selector:
            app: ow-docker-compose-${{ github.run_id }}
          type: LoadBalancer
          ports:
          - name: owgw-websocket
            port: 15002
            targetPort: 15002
            protocol: TCP
          - name: owgw-restapi
            port: 16002
            targetPort: 16002
            protocol: TCP
          - name: owgw-alivecheck
            port: 16102
            targetPort: 16102
            protocol: TCP
          - name: owgw-fileupload
            port: 16003
            targetPort: 16003
            protocol: TCP
          - name: owgw-ui-http
            port: 80
            targetPort: 80
            protocol: TCP
          - name: owgw-ui-https
            port: 443
            targetPort: 443
          - name: owsec-restapi
            port: 16001
            targetPort: 16001
            protocol: TCP
          - name: owsec-alivecheck
            port: 16101
            targetPort: 16101
          - name: owfms-restapi
            port: 16004
            targetPort: 16004
            protocol: TCP
          - name: owfms-alivecheck
            port: 16104
            targetPort: 16104
          - name: owprov-restapi
            port: 16005
            targetPort: 16005
            protocol: TCP
          - name: owprov-alivecheck
            port: 16105
            targetPort: 16105
          - name: owprov-ui-http
            port: 8080
            targetPort: 8080
            protocol: TCP
          - name: owprov-ui-https
            port: 8443
            targetPort: 8443
          - name: rttys-dev
            port: 5912
            targetPort: 5912
            protocol: TCP
          - name: rttys-user
            port: 5913
            targetPort: 5913
        ---
        apiVersion: v1
        kind: Pod
        metadata:
          name: ow-docker-compose-${{ github.run_id }}
          annotations:
            cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
          labels:
            app: ow-docker-compose-${{ github.run_id }}
        spec:
          volumes:
          - name: client-certs
            emptyDir: {}
          - name: deploy-repo
            emptyDir: {}
          - name: tip-cicd-wildcard-certs
            secret:
              secretName: tip-cicd-wildcard-certs
          containers:
          - name: docker-daemon
            image: docker:dind
            env:
            - name: DOCKER_TLS_CERTDIR
              value: "/certs"
            securityContext:
              privileged: true
            volumeMounts:
            - name: client-certs
              mountPath: /certs/client
            - name: deploy-repo
              mountPath: /wlan-cloud-ucentral-deploy
        
          - name: docker-compose-deployment
            image: docker/compose:latest
            command: ["/bin/sh", "-c"]
            args: ["apk add git; wget https://raw.githubusercontent.com/Telecominfraproject/wlan-testing/master/docker-compose/openwifi/deploy.sh; chmod +x deploy.sh; sh deploy.sh"]
            env:
            - name: DOCKER_TLS_VERIFY
              value: "true"
            - name: DOCKER_HOST
              value: "127.0.0.1:2376"
            - name: DOCKER_CERT_PATH
              value: "/certs/client"
            - name: OWGW_TAG
              value: "${{ github.event.inputs.owgw_version || github.event.client_payload.owgw_tag }}"
            - name: OWGWUI_TAG
              value: "${{ github.event.inputs.owgwui_version || github.event.client_payload.owgwui_tag }}"
            - name: OWSEC_TAG
              value: "${{ github.event.inputs.owsec_version || github.event.client_payload.owsec_tag }}"
            - name: OWFMS_TAG
              value: "${{ github.event.inputs.owfms_version || github.event.client_payload.owfms_tag }}"
            - name: OWPROV_TAG
              value: "${{ github.event.inputs.owprov_version || github.event.client_payload.owprov_tag }}"
            - name: OWPROVUI_TAG
              value: "${{ github.event.inputs.owprovui_version || github.event.client_payload.owprovui_tag }}"
            - name: INTERNAL_OWGW_HOSTNAME
              value: "owgw.wlan.local"
            - name: INTERNAL_OWSEC_HOSTNAME
              value: "owsec.wlan.local"
            - name: INTERNAL_OWFMS_HOSTNAME
              value: "owfms.wlan.local"
            - name: INTERNAL_OWPROV_HOSTNAME
              value: "owprov.wlan.local"
            - name: DEPLOY_VERSION
              value: "main"
            - name: SYSTEM_URI_UI
              value: "https://ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build"
            - name: DEFAULT_UCENTRALSEC_URL
              value: "https://ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build:16001"
            - name: RTTY_TOKEN
              value: "$RTTY_TOKEN"
            - name: OWGW_FILEUPLOADER_HOST_NAME
              value: "ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build"
            - name: OWGW_FILEUPLOADER_URI
              value: "https://ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build:16003"
            - name: OWGW_SYSTEM_URI_PRIVATE
              value: "https://owgw.wlan.local:17002"
            - name: OWGW_SYSTEM_URI_PUBLIC
              value: "https://ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build:16002"
            - name: OWGW_RTTY_SERVER
              value: "ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build"
            - name: OWSEC_AUTHENTICATION_DEFAULT_USERNAME
              value: "$OWSEC_AUTHENTICATION_DEFAULT_USERNAME"
            - name: OWSEC_AUTHENTICATION_DEFAULT_PASSWORD
              value: "$OWSEC_AUTHENTICATION_DEFAULT_PASSWORD"
            - name: OWSEC_SYSTEM_URI_PRIVATE
              value: "https://owsec.wlan.local:17001"
            - name: OWSEC_SYSTEM_URI_PUBLIC
              value: "https://ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build:16001"
            - name: OWFMS_SYSTEM_URI_PRIVATE
              value: "https://owfms.wlan.local:17004"
            - name: OWFMS_SYSTEM_URI_PUBLIC
              value: "https://ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build:16004"
            - name: OWFMS_S3_SECRET
              value: "$OWFMS_S3_SECRET"
            - name: OWFMS_S3_KEY
              value: "$OWFMS_S3_KEY"
            - name: OWPROV_SYSTEM_URI_PRIVATE
              value: "https://owprov.wlan.local:17005"
            - name: OWPROV_SYSTEM_URI_PUBLIC
              value: "https://ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build:16005"
            - name: WEBSOCKET_CERT
              value: "/tip-cicd-wildcard-certs/websocket-cert.pem"
            - name: WEBSOCKET_KEY
              value: "/tip-cicd-wildcard-certs/websocket-key.pem"
            volumeMounts:
            - name: client-certs
              mountPath: /certs/client
            - name: deploy-repo
              mountPath: /wlan-cloud-ucentral-deploy
            - name: tip-cicd-wildcard-certs
              readOnly: true
              mountPath: "/tip-cicd-wildcard-certs"
        EOF

    - name: Show resource state on deployment failure
      if: failure()
      run: |
        kubectl get pods --namespace ow-docker-compose-${{ github.run_id }}
        kubectl get services --namespace ow-docker-compose-${{ github.run_id }}
    - name: Describe pods on deployment failure
      if: failure()
      run: |
        kubectl describe pods --namespace ow-docker-compose-${{ github.run_id }}
    - name: Describe services on deployment failure
      if: failure()
      run: |
        kubectl describe services --namespace ow-docker-compose-${{ github.run_id }}

    - name: Change password and check if deployment is ready using clustersysteminfo image
      env:
        OWSEC: "ow-docker-compose-${{ github.run_id }}.cicd.lab.wlan.tip.build:16001"
        OWSEC_DEFAULT_USERNAME: ${{ secrets.OWSEC_DEFAULT_USERNAME }}
        OWSEC_DEFAULT_PASSWORD: ${{ secrets.OWSEC_DEFAULT_PASSWORD }}
        OWSEC_NEW_PASSWORD: ${{ secrets.OWSEC_NEW_PASSWORD }}
      run: |
        docker run --rm \
          -e OWSEC \
          -e OWSEC_DEFAULT_USERNAME \
          -e OWSEC_DEFAULT_PASSWORD \
          -e OWSEC_NEW_PASSWORD \
          tip-tip-wlan-cloud-ucentral.jfrog.io/clustersysteminfo:main

  test:
    runs-on: ubuntu-latest
    needs: [ build, deploy-controller ]
    steps:
    - name: get EKS access credentials
      run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }}

    - name: install Allure CLI tool
      run: |
        wget https://repo.maven.apache.org/maven2/io/qameta/allure/allure-commandline/${{ env.ALLURE_CLI_VERSION }}/allure-commandline-${{ env.ALLURE_CLI_VERSION }}.tgz
        tar -xzf allure-commandline-${{ env.ALLURE_CLI_VERSION }}.tgz

    - name: set job name
      id: job
      run: echo "::set-output name=name::testing-${{ github.run_number }}"

    - name: prepare namespace
      id: namespace
      run: |
        NAMESPACE="testing-${{ github.run_id }}"
        kubectl create ns $NAMESPACE
        kubectl config set-context --current --namespace=$NAMESPACE
        echo "::set-output name=name::${NAMESPACE}"

    - name: create configuration.py secret
      run: |
        cat << EOF > configuration.py
        ${{ secrets.LAB_CONFIGURATION }}
        EOF
        sed -i -r "s/'url': ('|\").*('|\")/'url': 'https:\/\/${{ needs.deploy-controller.outputs.sec_url }}:16001'/" configuration.py
        sed -i -r "s/'password': ('|\")openwifi('|\")/'password': '"${{ secrets.OWSEC_NEW_PASSWORD }}"'/" configuration.py
        cat configuration.py
        kubectl create secret generic configuration --from-file=configuration=./configuration.py

    - name: run tests
      run: |
        cat <<EOF | kubectl apply -f -
        apiVersion: batch/v1
        kind: Job
        metadata:
          name: "${{ steps.job.outputs.name }}"
        spec:
          template:
            metadata:
              annotations:
                cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
            spec:
              containers:
              - name: tests
                image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
                resources:
                  requests:
                    memory: "512Mi"
                    cpu: "250m"
                  limits:
                    memory: "512Mi"
                    cpu: "250m"
                command:
                  - /bin/bash
                  - -x
                  - -c
                  - |
                    cd tests
                    pytest -m "${{ github.event.inputs.marker_expression || 'sdk_restapi' }}" -s -vvv --testbed="${{ github.event.inputs.testbeds || 'basic-05' }}" --alluredir=/tmp/allure-results
                    ret=\$?
                    # sleep some time to be able to download the Allure results
                    sleep 60
                    exit \$ret
                volumeMounts:
                - name: configuration
                  mountPath: "/wlan-testing/tests/configuration.py"
                  subPath: configuration
                  readOnly: true
              nodeSelector:
                env: tests
              tolerations:
              - key: "tests"
                operator: "Exists"
                effect: "NoSchedule"
              imagePullSecrets:
              - name: tip-docker-registry-key
              restartPolicy: Never
              volumes:
              - name: configuration
                secret:
                  secretName: configuration
          backoffLimit: 0
        EOF
        # wait for pod to spawn
        sleep 1
        podname=$(kubectl get pods --no-headers -o custom-columns=":metadata.name" -l job-name="${{ steps.job.outputs.name }}" | sed "s/pod\///")
        kubectl wait "pod/$podname" --for condition=ready --timeout=600s
        kubectl logs -f $podname &
        #sleep 30 # wait for the pod to come up
        until [ -s test_everything.xml ]
        do
          sleep 10
          kubectl cp $podname:/wlan-testing/tests/test_everything.xml test_everything.xml >/dev/null 2>&1
        done
        echo "tests completed"
        echo "downloading allure results..."
        kubectl cp $podname:/tmp/allure-results allure-results >/dev/null 2>&1
        echo "waiting for pod to exit"
        kubectl logs -f $podname >/dev/null 2>&1
        exit $(kubectl get pod $podname --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}")

    - name: print logs
      if: always()
      run: |
        podname=$(kubectl get pods --no-headers -o custom-columns=":metadata.name" -l job-name="${{ steps.job.outputs.name }}" | sed "s/pod\///")
        kubectl logs $podname

    - name: upload Allure results as artifact
      if: always()
      uses: actions/upload-artifact@v2
      with:
        name: allure-results-docker-compose-${{ github.event.inputs.marker_expression || 'sdk_restapi' }}
        path: allure-results

    - name: cleanup
      if: always()
      run: |
        kubectl delete ns "${{ steps.namespace.outputs.name }}" --wait=true

  report:
    runs-on: ubuntu-latest
    needs: [ test ]
    if: always()
    steps:
    - name: checkout testing repo
      uses: actions/checkout@v2

    - uses: actions/download-artifact@v2
      with:
        name: allure-results-docker-compose-${{ github.event.inputs.marker_expression || 'sdk_restapi' }}
        path: allure-results

    - name: download history of previous run
      continue-on-error: true
      run: |
        LAST_RUN_ID=$(aws s3api head-object --bucket openwifi-allure-reports --key docker-compose-${{ github.event.inputs.marker_expression || 'sdk_restapi' }}/latest/index.html | jq -r .Metadata.latest)
        aws s3 cp --recursive s3://openwifi-allure-reports/docker-compose-${{ github.event.inputs.marker_expression || 'sdk_restapi' }}/$LAST_RUN_ID/history history

    - name: generate Allure report
      uses: ./.github/actions/generate-allure-report
      with:
        results_path: ./allure-results
        history_path: ./history

    - name: upload Allure report as artifact
      uses: actions/upload-artifact@v2
      with:
        name: allure-report-docker-compose-${{ github.event.inputs.marker_expression || 'sdk_restapi' }}
        path: allure-report

    # doing this to be able to aggregate multiple reports together later on
    - name: copy results into report
      run: |
        cp -r allure-results allure-report/results

    - name: upload to S3
      if: github.ref == 'refs/heads/master'
      uses: ./.github/actions/allure-report-to-s3
      with:
        test_type: docker-compose-${{ github.event.inputs.marker_expression || 'sdk_restapi' }}
        testbed: ${{ github.event.inputs.testbeds || 'basic-05' }}
        report_path: allure-report
        s3_access_key_id: ${{ secrets.ALLURE_S3_ACCESS_KEY_ID }}
        s3_access_key_secret: ${{ secrets.ALLURE_S3_ACCESS_KEY_SECRET }}

  delete-docker-image:
    needs: [ test ]
    runs-on: ubuntu-latest
    if: always()
    steps:
    - uses: actions/checkout@v2
    - name: cleanup Docker image
      uses: ./.github/actions/cleanup-docker
      with:
        registry_user: wlan-testing-cicd
        registry_password: ${{ secrets.DOCKER_USER_PASSWORD }}

#    - name: cleanup docker-compose-deployment image
#      run: curl -u${{ env.DOCKER_USER_NAME }}:${{ env.DOCKER_USER_PASSWORD }} -X DELETE "https://tip.jfrog.io/artifactory/tip-wlan-cloud-docker-repo/docker-compose-deployment/${{ github.run_id }}"

  destroy-controller:
    needs: [ test ]
    runs-on: ubuntu-latest
    if: always()
    steps:
    - name: Checkout wlan-testing repo
      uses: actions/checkout@v2
      with:
        path: wlan-testing

    - name: Fetch kubeconfig
      run: |
        aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }}

    - name: Delete namespace
      run: kubectl delete ns --ignore-not-found=true "ow-docker-compose-${{ github.run_id }}"