diff --git a/.allure-overview/generate_overview.py b/.allure-overview/generate_overview.py new file mode 100644 index 000000000..9490b50f0 --- /dev/null +++ b/.allure-overview/generate_overview.py @@ -0,0 +1,19 @@ +import json +import os +import pathlib +import string +import sys + +base = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) + +def main(): + with open(base / 'index.html', 'r') as f: + overview_template = string.Template(f.read()) + + with open(base / 'overviews.json', 'r') as f: + overview_config = json.load(f)[sys.argv[1]] + + print(overview_template.safe_substitute(**overview_config)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/.allure-overview/index.html b/.allure-overview/index.html new file mode 100644 index 000000000..f304a188c --- /dev/null +++ b/.allure-overview/index.html @@ -0,0 +1,52 @@ + + + + + + + + + +
+
+ +
+

+
+
+
+ + + + + \ No newline at end of file diff --git a/.allure-overview/overviews.json b/.allure-overview/overviews.json new file mode 100644 index 000000000..8652be812 --- /dev/null +++ b/.allure-overview/overviews.json @@ -0,0 +1,20 @@ +{ + "sanity": { + "OVERVIEW_TITLE": "'OpenWifi sanity results'", + "OVERVIEW_TESTBEDS": ["hfcl_ion4","tp-link_ec420-g1","edgecore_ecw5410","edgecore_ecw5211","edgecore_eap101","edgecore_eap102","cig_wf194c","cig_wf188n","indio_um-305ac"] + }, + "interop": { + "OVERVIEW_TITLE": "'OpenWifi interop results'", + "OVERVIEW_TESTBEDS": ["pixel-4","galaxy-s9","galaxy-s10","galaxy-s20","iphone-xr","iphone-7","iphone-11","iphone-12"] + }, + + "advanced": { + "OVERVIEW_TITLE": "'OpenWifi advanced results'", + "OVERVIEW_TESTBEDS": ["advanced-01","advanced-02"] + }, + + "performance": { + "OVERVIEW_TITLE": "'OpenWifi performance results'", + "OVERVIEW_TESTBEDS": ["basic-01","basic-02","basic-03","basic-04","basic-05","basic-06","basic-07","basic-08"] + } +} \ No newline at end of file diff --git a/.github/workflows/update-overviews.yml b/.github/workflows/update-overviews.yml new file mode 100644 index 000000000..1941b0f44 --- /dev/null +++ b/.github/workflows/update-overviews.yml @@ -0,0 +1,36 @@ +name: Update test report overview pages + +on: + push: + paths: + - ".allure-overview/**" + +jobs: + update_overview: + strategy: + fail-fast: false + matrix: + test_type: ['sanity', 'interop', 'advanced', 'performance'] + + runs-on: ubuntu-latest + container: python:3.8 + steps: + - uses: actions/checkout@v2 + + - name: generate overview page + run: | + python .allure-overview/generate_overview.py ${{ matrix.test_type }} > index.html + cat index.html + + - name: upload overview page + if: github.ref == 'refs/heads/master' + env: + AWS_S3_BUCKET_NAME: openwifi-allure-reports + AWS_ACCOUNT_ID: "289708231103" + AWS_ACCESS_KEY_ID: ${{ secrets.ALLURE_S3_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ALLURE_S3_ACCESS_KEY_SECRET }} + AWS_DEFAULT_REGION: us-east-1 + AWS_DEFAULT_OUTPUT: json + run: aws s3 cp --acl public-read --content-type text/html index.html s3://${AWS_S3_BUCKET_NAME}/${{ matrix.test_type }}/overview/index.html + + diff --git a/README.md b/README.md index e7c72499b..b61cc8589 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,11 @@ Using Swagger Autogenerated CloudSDK Library pypi package (implemented with [swa ### Reporting -Currently the plan is to use pytest integrated with [allure](https://docs.qameta.io/allure/#_pytest) to create visual reports for the test outcomes +Currently we use the [pytest allure integration](https://docs.qameta.io/allure/#_pytest) to create visual reports for the test outcomes. + +Additionally we generate overview pages, e.g. like [this one](http://openwifi-allure-reports.s3-website-us-east-1.amazonaws.com/sanity/overview/) to consolidate multiple test reports of the same test type. + +These overview pages are being deployed by [this workflow](.github/workflows/update-overviews.yml) and their configuration is set in [this file](./.allure-overview/overviews.json). ### Miscellaneous