diff --git a/.github/tools/generate_directory_index.py b/.github/tools/generate_directory_index.py new file mode 100644 index 000000000..98c8ef0f7 --- /dev/null +++ b/.github/tools/generate_directory_index.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +# --- +# Copyright 2020 glowinthedark +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and limitations under the License. +# --- +# +# Generate index.html files for +# all subdirectories in a directory tree. + +# -handle symlinked files and folders: displayed with custom icons + +# By default only the current folder is processed. + +# Use -r or --recursive to process nested folders. + +import argparse +import datetime +import os +import sys +from pathlib import Path +from urllib.parse import quote + +DEFAULT_OUTPUT_FILE = 'index.html' +EXCLUDED_PATHS = ['.git'] + + +def process_dir(top_dir, opts): + path_top_dir: Path + path_top_dir = Path(top_dir) + index_file = None + + index_path = Path(path_top_dir, opts.output_file) + + if opts.verbose: + print(f'Traversing dir {path_top_dir.absolute()}') + + # skip allure report dirs + if os.path.exists(index_path) and os.path.exists(Path(path_top_dir, 'history')) and os.path.exists(Path(path_top_dir, 'data')): + if opts.verbose: + print(f'Skipping Allure test report dir {path_top_dir.absolute()}') + return + + try: + index_file = open(index_path, 'w') + except Exception as e: + print('cannot create file %s %s' % (index_path, e)) + return + + index_file.write(""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

""" + f'{path_top_dir.name}' + """

+
+
+
+ + + + + + + + + + + + + + + + + + + """) + + # sort dirs first + sorted_entries = sorted(path_top_dir.glob('*'), key=lambda p: (p.is_file(), p.name), reverse=True) + + entry: Path + for entry in sorted_entries: + + # don't include index.html in the file listing + if entry.name.lower() == opts.output_file.lower(): + continue + + if entry.name.lower() in EXCLUDED_PATHS: + continue + + if entry.is_dir() and opts.recursive: + process_dir(entry, opts) + + # From Python 3.6, os.access() accepts path-like objects + if (not entry.is_symlink()) and not os.access(str(entry), os.W_OK): + print(f"*** WARNING *** entry {entry.absolute()} is not writable! SKIPPING!") + continue + if opts.verbose: + print(f'{entry.absolute()}') + + size_bytes = -1 ## is a folder + size_pretty = '—' + last_modified = '-' + last_modified_human_readable = '-' + last_modified_iso = '' + try: + if entry.is_file(): + size_bytes = entry.stat().st_size + size_pretty = pretty_size(size_bytes) + + if entry.is_dir() or entry.is_file(): + last_modified = datetime.datetime.fromtimestamp(entry.stat().st_mtime).replace(microsecond=0) + last_modified_iso = last_modified.isoformat() + last_modified_human_readable = last_modified.strftime("%c") + + except Exception as e: + print('ERROR accessing file name:', e, entry) + continue + + entry_path = str(entry.name) + + if entry.is_dir() and not entry.is_symlink(): + entry_type = 'folder' + if os.name not in ('nt',): + # append trailing slash to dirs, unless it's windows + entry_path = os.path.join(entry.name, '') + + elif entry.is_dir() and entry.is_symlink(): + entry_type = 'folder-shortcut' + print('dir-symlink', entry.absolute()) + + elif entry.is_file() and entry.is_symlink(): + entry_type = 'file-shortcut' + print('file-symlink', entry.absolute()) + + else: + entry_type = 'file' + + index_file.write(f""" + + + + + + + +""") + + index_file.write(""" + +
NameSize + Modified +
+ ..——
+ + + {entry.name} + + {size_pretty}
+
+
+ +""") + if index_file: + index_file.close() + + +# bytes pretty-printing +UNITS_MAPPING = [ + (1024 ** 5, ' PB'), + (1024 ** 4, ' TB'), + (1024 ** 3, ' GB'), + (1024 ** 2, ' MB'), + (1024 ** 1, ' KB'), + (1024 ** 0, (' byte', ' bytes')), +] + + +def pretty_size(bytes, units=UNITS_MAPPING): + """Human-readable file sizes. + + ripped from https://pypi.python.org/pypi/hurry.filesize/ + """ + for factor, suffix in units: + if bytes >= factor: + break + amount = int(bytes / factor) + + if isinstance(suffix, tuple): + singular, multiple = suffix + if amount == 1: + suffix = singular + else: + suffix = multiple + return str(amount) + suffix + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='''DESCRIPTION: + Generate directory index files (recursive is OFF by default). + Start from current dir or from folder passed as first positional argument. + Optionally filter by file types with --filter "*.py". ''') + + parser.add_argument('top_dir', + nargs='?', + action='store', + help='top folder from which to start generating indexes, ' + 'use current folder if not specified', + default=os.getcwd()) + + parser.add_argument('--output-file', '-o', + metavar='filename', + default=DEFAULT_OUTPUT_FILE, + help=f'Custom output file, by default "{DEFAULT_OUTPUT_FILE}"') + + parser.add_argument('--recursive', '-r', + action='store_true', + help="recursively process nested dirs (FALSE by default)", + required=False) + + parser.add_argument('--verbose', '-v', + action='store_true', + help='***WARNING: can take longer time with complex file tree structures on slow terminals***' + ' verbosely list every processed file', + required=False) + + config = parser.parse_args(sys.argv[1:]) + process_dir(config.top_dir, config) \ No newline at end of file diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 953e7015a..1f3d87775 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -160,7 +160,7 @@ jobs: - -c - | cd tests - pytest -m "${{ github.event.inputs.marker_expression }}" -s -vvv --testbed="${{ github.event.inputs.testbed }}" --skip-testrail --alluredir=/tmp/allure-results + pytest -m "${{ github.event.inputs.marker_expression || 'sanity' }}" -s -vvv --testbed="${{ github.event.inputs.testbed || 'basic-02' }}" --skip-testrail --alluredir=/tmp/allure-results ret=\$? # sleep some time to be able to download the Allure results sleep 60 @@ -201,31 +201,102 @@ jobs: exit $(kubectl get pod $podname --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}") - name: print logs - if: ${{ always() }} + if: always() run: | podname=$(kubectl get pods --no-headers -o custom-columns=":metadata.name" -l job-name="${{ steps.job.outputs.name }}" | sed "s/pod\///") kubectl logs $podname - name: upload Allure results as artifact - if: ${{ always() }} + if: always() uses: actions/upload-artifact@v2 with: name: allure-results path: allure-results + - name: cleanup + if: always() + run: | + kubectl delete job "${{ steps.job.outputs.name }}" --wait=true --ignore-not-found=true + kubectl delete secret configuration --wait=true --ignore-not-found=true + + report: + needs: [ test ] + if: always() + runs-on: ubuntu-latest + steps: + - name: install Allure CLI tool + run: | + wget https://repo.maven.apache.org/maven2/io/qameta/allure/allure-commandline/${{ env.ALLURE_CLI_VERSION }}/allure-commandline-${{ env.ALLURE_CLI_VERSION }}.tgz + tar -xzf allure-commandline-${{ env.ALLURE_CLI_VERSION }}.tgz + + - uses: actions/download-artifact@v2 + with: + name: allure-results + path: allure-results + + - name: checkout testing repo + uses: actions/checkout@v2 + with: + path: wlan-testing + + - name: get reports branch + uses: actions/checkout@v2 + continue-on-error: true + with: + ref: gh-pages + path: reports + + - name: copy history into results + run: | + if [ -e "reports/sanity/latest" ] ; then + cp -r reports/sanity/latest/history/ allure-results/history + fi + + - name: add report metadata + run: | + cat << EOF >> allure-results/environment.properties + Testbed=${{ github.event.inputs.testbed || 'basic-02' }} + Tests.CommitId=$(cd wlan-testing && git rev-parse --short HEAD) + CiRun.Id=${{ github.run_id }} + CiRun.Number=${{ github.run_number }} + CiRun.Url=https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + EOF + - name: generate Allure report - if: ${{ always() }} run: allure-${{ env.ALLURE_CLI_VERSION }}/bin/allure generate - name: upload Allure report as artifact - if: ${{ always() }} uses: actions/upload-artifact@v2 with: name: allure-report path: allure-report - - - name: cleanup - if: ${{ always() }} + + - name: copy new report + if: ${{ (github.event.inputs.marker_expression || 'sanity') == 'sanity' }} run: | - kubectl delete job "${{ steps.job.outputs.name }}" --wait=true --ignore-not-found=true - kubectl delete secret configuration --wait=true --ignore-not-found=true + mkdir -p reports/sanity + cp -Tr allure-report reports/sanity/${{ github.run_number }} + + - name: update latest symlink + if: ${{ (github.event.inputs.marker_expression || 'sanity') == 'sanity' }} + working-directory: reports/sanity + run: ln -fns ${{ github.run_number }} latest + + - name: generate new index.html + run: python wlan-testing/.github/tools/generate_directory_index.py -r reports + + - name: commit reports update + working-directory: reports + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + + git add . + git commit -m "Automated deployment: $(date -u)" + + - name: push + uses: ad-m/github-push-action@v0.6.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: gh-pages + directory: reports