From 5a0899b41c600a9131ddff8cf11afa924510ca2b Mon Sep 17 00:00:00 2001 From: Matthew Stidham Date: Fri, 30 Jul 2021 10:11:48 -0700 Subject: [PATCH] Make Ghost Request more versatile as kpi.csv files change so it doesn't break on older lanforge builds, and so we can add in new columns without difficulty. Signed-off-by: Matthew Stidham --- py-dashboard/GhostRequest.py | 157 +++++++++++++++------------------ py-dashboard/GrafanaRequest.py | 4 +- 2 files changed, 73 insertions(+), 88 deletions(-) diff --git a/py-dashboard/GhostRequest.py b/py-dashboard/GhostRequest.py index 36f28ab1..9792b42a 100644 --- a/py-dashboard/GhostRequest.py +++ b/py-dashboard/GhostRequest.py @@ -15,7 +15,6 @@ import requests import jwt from datetime import datetime -from dateutil import tz import json import subprocess from scp import SCPClient @@ -25,6 +24,7 @@ from influx2 import RecordInflux import time from collections import Counter import shutil +import itertools class CSVReader: @@ -36,8 +36,6 @@ class CSVReader: for x in df: if len(x) > 0: rows.append(x.split(sep)) - length = list(range(0, len(df[0]))) - columns = dict(zip(df[0], length)) return rows def get_column(self, @@ -74,7 +72,7 @@ class CSVReader: for row in df: for item in row: html = html + ('%s' % item) - html = html + ('\n') + html = html + '\n' html = html + ('' '') return html @@ -83,35 +81,24 @@ class CSVReader: target_index = df[0].index(column) counter = 0 targets = [0] - for row in df: + for row in df[1:]: try: if expression == 'less than': if float(row[target_index]) < target: targets.append(counter) - counter += 1 - else: - counter += 1 if expression == 'greater than': if float(row[target_index]) > target: targets.append(counter) - counter += 1 - else: - counter += 1 if expression == 'greater than or equal to': if float(row[target_index]) >= target: targets.append(counter) - counter += 1 - else: - counter += 1 - except: - counter += 1 + finally: + pass + counter += 1 return list(map(df.__getitem__, targets)) def concat(self, dfs): - final_df = dfs[0] - for df in dfs[1:]: - final_df = final_df + df[1:] - return final_df + return list(itertools.chain.from_iterable(dfs)) class GhostRequest: @@ -165,8 +152,6 @@ class GhostRequest: def create_post(self, title=None, text=None, - tags=None, - authors=None, status="published"): ghost_json_url = self.ghost_json_url + '/admin/posts/?source=html' post = dict() @@ -199,9 +184,9 @@ class GhostRequest: ghost_json_url = self.ghost_json_url + '/admin/images/upload/' token = self.encode_token() - bashCommand = "curl -X POST -F 'file=@%s' -H \"Authorization: Ghost %s\" %s" % (image, token, ghost_json_url) + bash_command = "curl -X POST -F 'file=@%s' -H \"Authorization: Ghost %s\" %s" % (image, token, ghost_json_url) - proc = subprocess.Popen(bashCommand, shell=True, stdout=subprocess.PIPE) + proc = subprocess.Popen(bash_command, shell=True, stdout=subprocess.PIPE) output = proc.stdout.read().decode('utf-8') if self.debug: print(output) @@ -226,12 +211,7 @@ class GhostRequest: head = head + '' % picture head = head + '''

This is the end of the example

''' self.create_post(title=title, - text=head, - tags='custom', - authors=authors) - - def list_append(self, list_1, value): - list_1.append(value) + text=head) def kpi_to_ghost(self, authors, @@ -258,11 +238,6 @@ class GhostRequest: text = '' csvreader = CSVReader() - if grafana_token is not None: - grafana = GrafanaRequest(grafana_token, - grafana_host, - grafanajson_port=grafana_port - ) if self.debug: print('Folders: %s' % folders) @@ -296,7 +271,7 @@ class GhostRequest: target_folders.append(folder) testbeds = list() - webpagesandpdfs = list() + web_pages_and_pdfs = list() high_priority_list = list() low_priority_list = list() images = list() @@ -306,23 +281,27 @@ class GhostRequest: subtest_pass_total = 0 subtest_fail_total = 0 test_tag = dict() + columns = ['test-rig', 'dut-hw-version', 'dut-sw-version', + 'dut-model-num', 'dut-serial-num'] + duts = dict() for target_folder in target_folders: try: target_file = '%s/kpi.csv' % target_folder df = csvreader.read_csv(file=target_file, sep='\t') - test_rig = csvreader.get_column(df, 'test-rig')[0] test_id = csvreader.get_column(df, 'test-id')[0] + for column in columns: + try: + column_data = csvreader.get_column(df, column)[0] + duts[column] = column_data + except: + print('no column named %s' % column) try: - test_tag[test_id] = (csvreader.get_column(df, 'test-tag')[0]) + test_tag[test_id] = csvreader.get_column(df, 'test-tag')[0] except: - notesttag = True + print('no test-id') pass_fail = Counter(csvreader.get_column(df, 'pass/fail')) test_pass_fail.append(pass_fail) - dut_hw = csvreader.get_column(df, 'dut-hw-version')[0] - dut_sw = csvreader.get_column(df, 'dut-sw-version')[0] - dut_model = csvreader.get_column(df, 'dut-model-num')[0] - dut_serial = csvreader.get_column(df, 'dut-serial-num')[0] subtest_pass = csvreader.get_column(df, 'Subtest-Pass') subtest_fail = csvreader.get_column(df, 'Subtest-Fail') for result in subtest_pass: @@ -333,10 +312,6 @@ class GhostRequest: subtest_pass_fail_list['PASS'] = subtest_pass_total subtest_pass_fail_list['FAIL'] = subtest_fail_total subtest_pass_fail.append(subtest_pass_fail_list) - if notesttag: - duts = [dut_serial, dut_hw, dut_sw, dut_model, test_rig, test_tag] - else: - duts = [dut_serial, dut_hw, dut_sw, dut_model, test_rig] times_append = csvreader.get_column(df, 'Date') if len(times_append) == 0: print(LookupError("%s/kpi.csv has no time points" % target_folder)) @@ -352,13 +327,12 @@ class GhostRequest: text = text + 'Tests passed: 0
' \ 'Tests failed : 0
' \ 'Percentage of tests passed: Not Applicable
' - testbeds.append(test_rig) + testbeds.append(duts['test-rig']) if testbed is None: - testbed = test_rig + testbed = duts['test-rig'] if test_run is None: test_run = now.strftime('%B-%d-%Y-%I-%M-%p-report') - local_path = '/home/%s/%s/%s/%s' % (user_push, customer, testbed, test_run) transport = paramiko.Transport(ghost_host, port) @@ -391,9 +365,9 @@ class GhostRequest: url = 'http://%s/%s/%s/%s/%s/%s' % ( ghost_host, customer.strip('/'), testbed, test_run, target_folder, 'index.html') webpages.append('HTML' % url) - webpagesandpdfsappend = dict() - webpagesandpdfsappend[test_id] = pdfs + webpages - webpagesandpdfs.append(webpagesandpdfsappend) + web_pages_and_pdfs_append = dict() + web_pages_and_pdfs_append[test_id] = pdfs + webpages + web_pages_and_pdfs.append(web_pages_and_pdfs_append) scp_push.close() self.upload_images(target_folder) for image in self.images: @@ -413,9 +387,9 @@ class GhostRequest: pass low_priority = csvreader.filter_df(results, 'test-priority', 'less than', 94) + print('Low Priority results %s' % len(low_priority)) high_priority = csvreader.filter_df(results, 'test-priority', 'greater than or equal to', 95) high_priority_list.append(high_priority) - low_priority_list.append(low_priority) except: @@ -423,11 +397,10 @@ class GhostRequest: target_folders.remove(target_folder) failuredict = dict() failuredict[target_folder] = ['Failure'] - webpagesandpdfs.append(failuredict) + web_pages_and_pdfs.append(failuredict) if len(times) == 0: return ArithmeticError("There are no datapoints in any folders passed into Ghost") - test_pass_fail_results = sum((Counter(test) for test in test_pass_fail), Counter()) subtest_pass_fail_results = sum((Counter(test) for test in subtest_pass_fail), Counter()) @@ -443,14 +416,17 @@ class GhostRequest: high_priority = csvreader.concat(high_priority_list) low_priority = csvreader.concat(low_priority_list) - high_priority = csvreader.get_columns(high_priority, - ['Short Description', 'Score', 'Test Details']) + if len(high_priority) > 0: + high_priority = csvreader.get_columns(high_priority, + ['Short Description', 'Score', 'Test Details']) low_priority = csvreader.get_columns(low_priority, ['Short Description', 'Score', 'Test Details']) high_priority.append(['Total Passed', test_pass_fail_results['PASS'], 'Total subtests passed during this run']) high_priority.append(['Total Failed', test_pass_fail_results['FAIL'], 'Total subtests failed during this run']) - high_priority.append(['Subtests Passed', subtest_pass_fail_results['PASS'], 'Total subtests passed during this run']) - high_priority.append(['Subtests Failed', subtest_pass_fail_results['FAIL'], 'Total subtests failed during this run']) + high_priority.append( + ['Subtests Passed', subtest_pass_fail_results['PASS'], 'Total subtests passed during this run']) + high_priority.append( + ['Subtests Failed', subtest_pass_fail_results['FAIL'], 'Total subtests failed during this run']) if title is None: title = end_time.strftime('%B %d, %Y %I:%M %p report') @@ -461,15 +437,6 @@ class GhostRequest: target_files.append(folder.split('/')[-1] + '/kpi.csv') if self.debug: print('Target files: %s' % target_files) - grafana.create_custom_dashboard(target_csvs=target_files, - title=title, - datasource=grafana_datasource, - bucket=grafana_bucket, - from_date=start_time, - to_date=end_time.strftime('%Y-%m-%d %H:%M:%S'), - pass_fail='GhostRequest', - testbed=testbeds[0], - test_tag=test_tag) if self.influx_token is not None: influxdb = RecordInflux(_influx_host=self.influx_host, @@ -520,22 +487,28 @@ class GhostRequest: for tag in list(set(test_tag.values())): print(tag) test_tag_table += ( - 'Test Tag' \ - '%s' % tag) + 'Test Tag%s' % tag) + dut_table_column_names = {'test-rig': 'Testbed', + 'dut-hw-version': 'DUT HW', + 'dut-sw-version': 'DUT SW', + 'dut-model-num': 'DUT Model', + 'dut-serial-num': 'DUT Serial'} + dut_table_columns = '' + for column in columns: + if column in dut_table_column_names.keys(): + column_name = dut_table_column_names[column] + else: + column_name = column + dut_table_columns += ( + '%s%s' % ( + column_name, duts[column]) + ) + dut_table = '' \ '' \ - '' \ - '' \ '%s' \ - '' \ - '' \ - '' \ - '' \ - '' \ - '' \ - '' \ - '' \ + '%s' \ '' \ '' \ '' \ @@ -544,13 +517,13 @@ class GhostRequest: '' \ '' \ '' % ( - duts[4], test_tag_table, duts[1], duts[2], duts[3], duts[0], test_pass_fail_results['PASS'], + dut_table_columns, test_tag_table, test_pass_fail_results['PASS'], test_pass_fail_results['FAIL'], subtest_pass_total, subtest_fail_total) dut_table = dut_table + '
Test Information
Testbed%s
DUT_HW%s
DUT_SW%s
DUT model%s
DUT Serial%s
Tests passed%s
Tests failed%s
Subtests failed%s
' text = text + dut_table - for dictionary in webpagesandpdfs: + for dictionary in web_pages_and_pdfs: text += list(dictionary.keys())[0] + ' report: ' for value in dictionary.values(): for webpage in value: @@ -565,6 +538,20 @@ class GhostRequest: text = text + 'High priority results: %s' % csvreader.to_html(high_priority) if grafana_token is not None: + grafana = GrafanaRequest(grafana_token, + grafana_host, + grafanajson_port=grafana_port + ) + print(test_tag) + grafana.create_custom_dashboard(target_csvs=target_files, + title=title, + datasource=grafana_datasource, + bucket=grafana_bucket, + from_date=start_time, + to_date=end_time.strftime('%Y-%m-%d %H:%M:%S'), + pass_fail='GhostRequest', + testbed=testbeds[0], + test_tag=test_tag) # get the details of the dashboard through the API, and set the end date to the youngest KPI grafana.list_dashboards() @@ -577,6 +564,4 @@ class GhostRequest: text = text + 'Low priority results: %s' % csvreader.to_html(low_priority) self.create_post(title=title, - text=text, - tags='custom', - authors=authors) + text=text) diff --git a/py-dashboard/GrafanaRequest.py b/py-dashboard/GrafanaRequest.py index 302c29a0..e132b7c1 100644 --- a/py-dashboard/GrafanaRequest.py +++ b/py-dashboard/GrafanaRequest.py @@ -170,8 +170,8 @@ class GrafanaRequest: if graph_group is not None: graphgroup = ('|> filter(fn: (r) => r["Graph-Group"] == "%s")\n' % graph_group) query += graphgroup - if test_tag is not None: - graphgroup = ('|> filter(fn: (r) => r["Test-Tag"] == "%s")\n' % test_tag) + if test_tag is not None and len(test_tag) > 0: + graphgroup = ('|> filter(fn: (r) => r["Test Tag"] == "%s")\n' % test_tag) query += graphgroup if testbed is not None: query += ('|> filter(fn: (r) => r["testbed"] == "%s")\n' % testbed)