diff --git a/py-dashboard/GhostRequest.py b/py-dashboard/GhostRequest.py index 8ebaf300..1b576389 100644 --- a/py-dashboard/GhostRequest.py +++ b/py-dashboard/GhostRequest.py @@ -61,7 +61,7 @@ class CSVReader: def to_html(self, df): html = '' html = html + ('') - for row in df: + for row in df[1:]: for item in row: html = html + ('' % item) html = html + ('\n') @@ -75,12 +75,18 @@ class CSVReader: for row in df: try: if expression == 'less than': - if float(row[target_index]) <= target: + if float(row[target_index]) < target: targets.append(counter) counter += 1 else: counter += 1 if expression == 'greater than': + if float(row[target_index]) > target: + targets.append(counter) + counter += 1 + else: + counter += 1 + if expression == 'greater than or equal to': if float(row[target_index]) >= target: targets.append(counter) counter += 1 @@ -90,6 +96,12 @@ class CSVReader: counter += 1 return list(map(df.__getitem__, targets)) + def concat(self, dfs): + final_df = dfs[0] + for df in dfs[1:]: + final_df = final_df + df[1:] + return final_df + class GhostRequest: def __init__(self, @@ -193,25 +205,26 @@ class GhostRequest: tags='custom', authors=authors) - def wifi_capacity_to_ghost(self, - authors, - folders, - title=None, - server_pull=None, - ghost_host=None, - port='22', - user_pull='lanforge', - password_pull='lanforge', - user_push=None, - password_push=None, - customer=None, - testbed='Unknown Testbed', - test_run=None, - target_folders=list(), - grafana_dashboard=None, - grafana_token=None, - grafana_host=None, - grafana_port=3000): + def kpi_to_ghost(self, + authors, + folders, + parent_folder=None, + title=None, + server_pull=None, + ghost_host=None, + port=22, + user_push=None, + password_push=None, + customer=None, + testbed='Unknown Testbed', + test_run=None, + target_folders=list(), + grafana_dashboard=None, + grafana_token=None, + grafana_host=None, + grafana_port=3000, + grafana_datasource='InfluxDB', + grafana_bucket=None): text = '' csvreader = CSVReader() if grafana_token is not None: @@ -219,27 +232,44 @@ class GhostRequest: grafana_host, grafanajson_port=grafana_port ) - if test_run is None: - test_run = sorted(folders)[0].split('/')[-1].strip('/') - print(folders) - for folder in folders: - print(folder) - ssh_pull = paramiko.SSHClient() - ssh_pull.set_missing_host_key_policy(paramiko.client.AutoAddPolicy) - ssh_pull.connect(server_pull, - port, - username=user_pull, - password=password_pull, - allow_agent=False, - look_for_keys=False) - scp_pull = SCPClient(ssh_pull.get_transport()) - scp_pull.get(folder, recursive=True) - target_folder = str(folder).rstrip('/').split('/')[-1] - target_folders.append(target_folder) - print('Target folder: %s' % target_folder) + #if parent_folder is None: + #test_run = sorted(folders)[0].split('/')[-1].strip('/') + print('Folders: %s' % folders) + + ssh_push = paramiko.SSHClient() + ssh_push.set_missing_host_key_policy(paramiko.client.AutoAddPolicy) + ssh_push.connect(ghost_host, + port, + username=user_push, + password=password_push, + allow_agent=False, + look_for_keys=False) + scp_push = SCPClient(ssh_push.get_transport()) + + if parent_folder is not None: + print("parent_folder %s" % parent_folder) + files = os.listdir(parent_folder) + print(files) + for file in files: + if os.path.isdir(parent_folder+'/'+file) is True: + import shutil + shutil.move(parent_folder+'/'+file, file) + target_folders.append(file) + print('Target folders: %s' % target_folders) + else: + for folder in folders: + print(folder) + target_folders.append(folder) + + testbeds = list() + pdfs = list() + high_priority_list = list() + low_priority_list = list() + images = list() + + for target_folder in target_folders: try: target_file = '%s/kpi.csv' % target_folder - print('target file %s' % target_file) df = csvreader.read_csv(file=target_file, sep='\t') csv_testbed = csvreader.get_column(df, 'test-rig')[0] pass_fail = Counter(csvreader.get_column(df, 'pass/fail')) @@ -249,87 +279,94 @@ class GhostRequest: text = text + 'Percentage of tests passed: %s
' % ( pass_fail['PASS'] / (pass_fail['PASS'] + pass_fail['FAIL'])) - print(csv_testbed) except: - pass + print("Failure") + target_folders.remove(target_folder) + break if len(csv_testbed) > 2: testbed = csv_testbed - text = text + 'Testbed: %s
' % testbed + testbeds.append(testbed) if testbed == 'Unknown Testbed': raise UserWarning('Please define your testbed') - print('testbed %s' % testbed) - ssh_push = paramiko.SSHClient() - ssh_push.set_missing_host_key_policy(paramiko.client.AutoAddPolicy) - ssh_push.connect(ghost_host, - port, - username=user_push, - password=password_push, - allow_agent=False, - look_for_keys=False) - scp_push = SCPClient(ssh_push.get_transport()) local_path = '/home/%s/%s/%s' % (user_push, customer, testbed) - transport = paramiko.Transport((ghost_host, port)) + + transport = paramiko.Transport(ghost_host, port) transport.connect(None, user_push, password_push) sftp = paramiko.sftp_client.SFTPClient.from_transport(transport) - print('Local Path: %s' % local_path) - try: - sftp.mkdir(local_path) - except: - print('folder %s already exists' % local_path) - scp_push.put(target_folder, recursive=True, remote_path=local_path) - print(local_path + '/' + target_folder) + + print(local_path) + print(target_folder) + scp_push.put(target_folder, local_path, recursive=True) files = sftp.listdir(local_path + '/' + target_folder) - # print('Files: %s' % files) for file in files: if 'pdf' in file: - url = 'http://%s/%s/%s/%s/%s/%s' % ( - ghost_host, customer.strip('/'), testbed, test_run, target_folder, file) - text = text + 'PDF of results: %s
' % (url, file) - print(url) - scp_pull.close() + url = 'http://%s/%s/%s/%s/%s' % ( + ghost_host, customer.strip('/'), testbed, test_run, file) + pdfs.append('PDF of results: %s
' % (url, file)) scp_push.close() self.upload_images(target_folder) for image in self.images: if 'kpi-' in image: if '-print' not in image: - text = text + '' % image + images.append('' % image) self.images = [] - if grafana_token is not None: - # get the details of the dashboard through the API, and set the end date to the youngest KPI - grafana.list_dashboards() + results = csvreader.get_columns(df, ['short-description', 'numeric-score', 'test details', 'test-priority']) - grafana.create_snapshot(title=grafana_dashboard) - time.sleep(3) - snapshot = grafana.list_snapshots()[-1] - print(snapshot) - text = text + '
' % ( - grafana_host, snapshot['key'], '%') + low_priority = csvreader.filter_df(results, 'test-priority', 'less than', 94) + high_priority = csvreader.filter_df(results, 'test-priority', 'greater than or equal to', 95) + high_priority_list.append(high_priority) - results = csvreader.get_columns(df,['short-description','numeric-score','test details','test-priority']) - - low_priority = csvreader.to_html(csvreader.filter_df(results, 'test-priority', 'less than', 94)) - high_priority = csvreader.to_html(csvreader.filter_df(results, 'test-priority', 'greater than', 95)) - - text = text + 'High priority results: %s' % high_priority - - text = text + 'Low priority results: %s' % low_priority + low_priority_list.append(low_priority) now = date.now() + high_priority = csvreader.concat(high_priority_list) + low_priority = csvreader.concat(low_priority_list) + + high_priority = csvreader.get_columns(high_priority, ['short-description', 'numeric-score', 'test details']) + low_priority = csvreader.get_columns(low_priority, ['short-description', 'numeric-score', 'test details']) + if title is None: title = "%s %s %s %s:%s report" % (now.day, now.month, now.year, now.hour, now.minute) # create Grafana Dashboard target_files = [] - for folder in folders: - print(folder) + for folder in target_folders: target_files.append(folder.split('/')[-1] + '/kpi.csv') + print('Target files: %s' % target_files) grafana.create_custom_dashboard(target_csvs=target_files, - title=title) + title=title, + datasource=grafana_datasource, + bucket=grafana_bucket) + + try: + text = 'Testbed: %s
' % testbeds[0] + except: + text = '' + + for pdf in pdfs: + text = text + pdf + + for image in images: + text = text + image + + text = text + 'High priority results: %s' % csvreader.to_html(high_priority) + + if grafana_token is not None: + # get the details of the dashboard through the API, and set the end date to the youngest KPI + grafana.list_dashboards() + + grafana.create_snapshot(title='Testbed: ' + title) + time.sleep(3) + snapshot = grafana.list_snapshots()[-1] + text = text + '
' % ( + grafana_host, snapshot['key'], '%') + + text = text + 'Low priority results: %s' % csvreader.to_html(low_priority) self.create_post(title=title, text=text, tags='custom', - authors=authors) + authors=authors) \ No newline at end of file diff --git a/py-dashboard/GrafanaRequest.py b/py-dashboard/GrafanaRequest.py index 59ab188c..5aba9248 100644 --- a/py-dashboard/GrafanaRequest.py +++ b/py-dashboard/GrafanaRequest.py @@ -132,6 +132,49 @@ class GrafanaRequest: print(dictionary) return dictionary + def maketargets(self, + bucket, + scriptname, + groupBy, + index, + graph_group, + testbed): + query = ( + 'from(bucket: "%s")\n ' + '|> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n ' + '|> filter(fn: (r) => r["script"] == "%s")\n ' + '|> group(columns: ["_measurement"])\n ' + % (bucket, scriptname)) + queryend = ('|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n ' + '|> yield(name: "mean")\n ') + if graph_group is not None: + graphgroup = ('|> filter(fn: (r) => r["Graph-Group"] == "%s")\n' % graph_group) + query += graphgroup + if testbed is not None: + query += ('|> filter(fn: (r) => r["testbed"] == "%s")\n' % testbed) + targets = dict() + targets['delimiter'] = ',' + targets['groupBy'] = groupBy + targets['header'] = True + targets['ignoreUnknown'] = False + targets['orderByTime'] = 'ASC' + targets['policy'] = 'default' + targets['query'] = query + queryend + targets['refId'] = dict(enumerate(string.ascii_uppercase, 1))[index + 1] + targets['resultFormat'] = "time_series" + targets['schema'] = list() + targets['skipRows'] = 0 + targets['tags'] = list() + return targets + + def groupby(self, params, grouptype): + dic = dict() + dic['params'] = list() + dic['params'].append(params) + dic['type'] = grouptype + return dic + + def create_custom_dashboard(self, scripts=None, title=None, @@ -174,20 +217,10 @@ class GrafanaRequest: target_csvs = open(graph_groups_file).read().split('\n') graph_groups = self.get_graph_groups( target_csvs) # Get the list of graph groups which are in the tests we ran - unit_dict = dict() - for csv in target_csvs: - if len(csv) > 1: - print(csv) - unit_dict.update(self.get_units(csv)) if target_csvs: print('Target CSVs: %s' % target_csvs) graph_groups = self.get_graph_groups( target_csvs) # Get the list of graph groups which are in the tests we ran - unit_dict = dict() - for csv in target_csvs: - if len(csv) > 1: - print(csv) - unit_dict.update(self.get_units(csv)) for scriptname in graph_groups.keys(): for graph_group in graph_groups[scriptname]: panel = dict() @@ -210,14 +243,14 @@ class GrafanaRequest: options = dict() options['alertThreshold'] = True - #groupBy = list() - #groupBy.append(self.groupby('$__interval', 'time')) - #groupBy.append(self.groupby('null', 'fill')) + groupBy = list() + groupBy.append(self.groupby('$__interval', 'time')) + groupBy.append(self.groupby('null', 'fill')) - #targets = list() - #counter = 0 - #new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group, testbed) - #targets.append(new_target) + targets = list() + counter = 0 + new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group, testbed) + targets.append(new_target) fieldConfig = dict() fieldConfig['defaults'] = dict() @@ -274,7 +307,7 @@ class GrafanaRequest: panel['spaceLength'] = 10 panel['stack'] = False panel['steppedLine'] = False - #panel['targets'] = targets + panel['targets'] = targets panel['thresholds'] = list() panel['timeFrom'] = None panel['timeRegions'] = list() diff --git a/py-scripts/ghost_profile.py b/py-scripts/ghost_profile.py index 1dcb300d..3411fd71 100755 --- a/py-scripts/ghost_profile.py +++ b/py-scripts/ghost_profile.py @@ -6,17 +6,17 @@ PURPOSE: modify ghost database from the command line. SETUP: A Ghost installation which the user has admin access to. EXAMPLE: ./ghost_profile.py --article_text_file text.txt --title Test --authors Matthew --ghost_token SECRET_KEY --host 192.168.1.1 -There is a specific class for uploading wifi capacity graphs called wifi_capacity. +There is a specific class for uploading kpi graphs called kpi_to_ghost. EXAMPLE: ./ghost_profile.py --ghost_token TOKEN --ghost_host 192.168.100.147 --folders /home/lanforge/html-reports/wifi-capacity-2021-06-04-02-51-07 ---wifi_capacity appl --authors Matthew --title 'wifi capacity 2021 06 04 02 51 07' --server 192.168.93.51 +--kpi_to_ghost appl --authors Matthew --title 'wifi capacity 2021 06 04 02 51 07' --server 192.168.93.51 --user_pull lanforge --password_pull lanforge --customer candela --testbed heather --test_run test-run-6 --user_push matt --password_push PASSWORD EXAMPLE 2: ./ghost_profile.py --ghost_token TOKEN --ghost_host 192.168.100.147 --server 192.168.93.51 --user_pull lanforge --password_pull lanforge --customer candela ---testbed heather --user_push matt --password_push "amount%coverage;Online" --wifi_capacity app +--testbed heather --user_push matt --password_push "amount%coverage;Online" --kpi_to_ghost app --folders /home/lanforge/html-reports/wifi-capacity-2021-06-14-10-42-29 --grafana_token TOKEN --grafana_host 192.168.100.201 --grafana_dashboard 'Stidmatt-02' @@ -60,43 +60,45 @@ class UseGhost(GhostRequest): text = open(file).read() return self.create_post(title=title, text=text, tags=tags, authors=authors) - def wifi_capacity(self, - authors, - folders, - title, - server_pull, - ghost_host, - port, - user_pull, - password_pull, - user_push, - password_push, - customer, - testbed, - test_run, - grafana_dashboard, - grafana_token, - grafana_host, - grafana_port): + def kpi(self, + authors, + folders, + parent_folder, + title, + server_pull, + ghost_host, + port, + user_push, + password_push, + customer, + testbed, + test_run, + grafana_dashboard, + grafana_token, + grafana_host, + grafana_port, + datasource, + grafana_bucket): target_folders = list() - return self.wifi_capacity_to_ghost(authors, - folders, - title, - server_pull, - ghost_host, - port, - user_pull, - password_pull, - user_push, - password_push, - customer, - testbed, - test_run, - target_folders, - grafana_dashboard, - grafana_token, - grafana_host, - grafana_port) + return self.kpi_to_ghost(authors, + folders, + parent_folder, + title, + server_pull, + ghost_host, + port, + user_push, + password_push, + customer, + testbed, + test_run, + target_folders, + grafana_dashboard, + grafana_token, + grafana_host, + grafana_port, + datasource, + grafana_bucket) def main(): @@ -125,12 +127,10 @@ def main(): optional.add_argument('--image', default=None) optional.add_argument('--folder', default=None) optional.add_argument('--custom_post', default=None) - optional.add_argument('--wifi_capacity', default=None) + optional.add_argument('--kpi_to_ghost', help='Generate a Ghost report from KPI spreadsheets', action="store_true") optional.add_argument('--folders', action='append', default=None) optional.add_argument('--server_pull') optional.add_argument('--port', default=22) - optional.add_argument('--user_pull', default='lanforge') - optional.add_argument('--password_pull', default='lanforge') optional.add_argument('--user_push') optional.add_argument('--password_push') optional.add_argument('--customer') @@ -140,6 +140,9 @@ def main(): optional.add_argument('--grafana_token', default=None) optional.add_argument('--grafana_host', default=None) optional.add_argument('--grafana_port', default=3000) + optional.add_argument('--parent_folder', default=None) + optional.add_argument('--datasource', default='InfluxDB') + optional.add_argument('--grafana_bucket', default=None) optional.add_argument('--debug') args = parser.parse_args() @@ -165,24 +168,25 @@ def main(): if args.folder is not None: Ghost.upload_images(args.folder) - if args.wifi_capacity is not None: - Ghost.wifi_capacity(args.authors, - args.folders, - args.title, - args.server_pull, - args.ghost_host, - args.port, - args.user_pull, - args.password_pull, - args.user_push, - args.password_push, - args.customer, - args.testbed, - args.test_run, - args.grafana_dashboard, - args.grafana_token, - args.grafana_host, - args.grafana_port) + if args.kpi_to_ghost is True: + Ghost.kpi(args.authors, + args.folders, + args.parent_folder, + args.title, + args.server_pull, + args.ghost_host, + args.port, + args.user_push, + args.password_push, + args.customer, + args.testbed, + args.test_run, + args.grafana_dashboard, + args.grafana_token, + args.grafana_host, + args.grafana_port, + args.datasource, + args.grafana_bucket) if __name__ == "__main__": diff --git a/py-scripts/grafana_profile.py b/py-scripts/grafana_profile.py index 5abc1767..ad8bcc90 100755 --- a/py-scripts/grafana_profile.py +++ b/py-scripts/grafana_profile.py @@ -22,47 +22,6 @@ from LANforge.lfcli_base import LFCliBase import string class UseGrafana(GrafanaRequest): - def groupby(self, params, grouptype): - dic = dict() - dic['params'] = list() - dic['params'].append(params) - dic['type'] = grouptype - return dic - - def maketargets(self, - bucket, - scriptname, - groupBy, - index, - graph_group, - testbed): - query = ( - 'from(bucket: "%s")\n ' - '|> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n ' - '|> filter(fn: (r) => r["script"] == "%s")\n ' - '|> group(columns: ["_measurement"])\n ' - % (bucket, scriptname)) - queryend = ('|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n ' - '|> yield(name: "mean")\n ') - if graph_group is not None: - graphgroup = ('|> filter(fn: (r) => r["Graph-Group"] == "%s")\n' % graph_group) - query += graphgroup - if testbed is not None: - query += ('|> filter(fn: (r) => r["testbed"] == "%s")\n' % testbed) - targets = dict() - targets['delimiter'] = ',' - targets['groupBy'] = groupBy - targets['header'] = True - targets['ignoreUnknown'] = False - targets['orderByTime'] = 'ASC' - targets['policy'] = 'default' - targets['query'] = query + queryend - targets['refId'] = dict(enumerate(string.ascii_uppercase, 1))[index + 1] - targets['resultFormat'] = "time_series" - targets['schema'] = list() - targets['skipRows'] = 0 - targets['tags'] = list() - return targets def read_csv(self, file):
%s