From 5b1c188ac210c5aef36a58cbba16acfda798b91d Mon Sep 17 00:00:00 2001 From: Matthew Stidham Date: Tue, 15 Jun 2021 15:56:17 -0700 Subject: [PATCH] Include embedded dashboard in ghost page Signed-off-by: Matthew Stidham --- py-dashboard/GhostRequest.py | 99 ++++++++++++++++++++++------------ py-dashboard/GrafanaRequest.py | 56 +++++++++++++------ py-scripts/ghost_profile.py | 26 +++++++-- py-scripts/grafana_profile.py | 65 ++++++++++++---------- 4 files changed, 165 insertions(+), 81 deletions(-) diff --git a/py-dashboard/GhostRequest.py b/py-dashboard/GhostRequest.py index 5aa483aa..aa593cdc 100644 --- a/py-dashboard/GhostRequest.py +++ b/py-dashboard/GhostRequest.py @@ -19,22 +19,25 @@ import json import subprocess from scp import SCPClient import paramiko +from GrafanaRequest import GrafanaRequest class CSVReader: def read_csv(self, file, sep=','): - csv = open(file).read().split('\n') + df = open(file).read().split('\n') rows = list() - for x in csv: + for x in df: if len(x) > 0: rows.append(x.split(sep)) length = list(range(0, len(df[0]))) columns = dict(zip(df[0], length)) return rows - def get_column(df, value): + def get_column(self, + df, + value): index = df[0].index(value) values = [] for row in df[1:]: @@ -147,7 +150,7 @@ class GhostRequest: def wifi_capacity_to_ghost(self, authors, folders, - title='Wifi Capacity', + title=None, server_pull=None, ghost_host=None, port='22', @@ -156,21 +159,19 @@ class GhostRequest: user_push=None, password_push=None, customer=None, - testbed=None, - test_run=None): - text = '''The Candela WiFi Capacity test is designed to measure performance of an Access Point when handling - different amounts of WiFi Stations. The test allows the user to increase the number of stations in user - defined steps for each test iteration and measure the per station and the overall throughput for each trial. - Along with throughput other measurements made are client connection times, Fairness, % packet loss, - DHCP times and more. The expected behavior is for the AP to be able to handle several stations (within the - limitations of the AP specs) and make sure all stations get a fair amount of airtime both in the upstream and - downstream. An AP that scales well will not show a significant over-all throughput decrease as more stations - are added. - Realtime Graph shows summary download and upload RX bps of connections created by this test.
''' - + testbed='Unknown Testbed', + test_run=None, + target_folders=list(), + grafana_dashboard=None, + grafana_token=None, + grafana_host=None, + grafana_port=3000): + text = '' + csvreader = CSVReader() if test_run is None: - test_run = sorted(folders)[0].split('/')[-1] + test_run = sorted(folders)[0].split('/')[-1].strip('/') for folder in folders: + print(folder) ssh_pull = paramiko.SSHClient() ssh_pull.set_missing_host_key_policy(paramiko.client.AutoAddPolicy) ssh_pull.connect(server_pull, @@ -181,6 +182,24 @@ class GhostRequest: look_for_keys=False) scp_pull = SCPClient(ssh_pull.get_transport()) scp_pull.get(folder, recursive=True) + target_folder = str(folder).rstrip('/').split('/')[-1] + target_folders.append(target_folder) + print(target_folder) + try: + target_file = '%s/kpi.csv' % target_folder + print('target file %s' % target_file) + df = csvreader.read_csv(file=target_file, sep='\t') + csv_testbed = csvreader.get_column(df, 'test-rig')[0] + print(csv_testbed) + except: + pass + if len(csv_testbed) > 2: + testbed = csv_testbed + text = text + 'Testbed: %s
' % testbed + if testbed == 'Unknown Testbed': + raise UserWarning('Please define your testbed') + print('testbed %s' % testbed) + ssh_push = paramiko.SSHClient() ssh_push.set_missing_host_key_policy(paramiko.client.AutoAddPolicy) ssh_push.connect(ghost_host, @@ -195,32 +214,44 @@ class GhostRequest: transport.connect(None, user_push, password_push) sftp = paramiko.sftp_client.SFTPClient.from_transport(transport) print(local_path) - sftp.mkdir(local_path) - target_folder = folder.split('/')[-1] - print(target_folder) + try: + sftp.mkdir(local_path) + except: + print('folder %s already exists' % local_path) scp_push.put(target_folder, recursive=True, remote_path=local_path) files = sftp.listdir(local_path + '/' + target_folder) - print('Files: %s' % files) + # print('Files: %s' % files) for file in files: if 'pdf' in file: - self.pdfs.append(file) - - #df = CSVReader.read_csv(local_path + '/' + target_folder + '/' + 'kpi.csv') - #testbed = CSVReader.get_column(df, 'test-rig')[0] - - print('PDFs %s' % self.pdfs) + url = 'http://%s/%s/%s/%s/%s/%s' % ( + ghost_host, customer.strip('/'), testbed, test_run, target_folder, file) + text = text + 'PDF of results: %s
' % (url, file) + print(url) scp_pull.close() scp_push.close() self.upload_images(target_folder) + for image in self.images: + if 'kpi-' in image: + if '-print' not in image: + text = text + '' % image + self.images = [] - for pdf in self.pdfs: - url = 'http://%s/%s/%s/%s/%s/%s' % (ghost_host, customer, testbed, test_run, target_folder, pdf) - text = text + 'PDF of results: %s' % (url, url) + if grafana_token is not None: + GR = GrafanaRequest(grafana_token, + grafana_host, + grafanajson_port=grafana_port + ) + GR.create_snapshot(title=grafana_dashboard) + snapshot = GR.list_snapshots()[-1] + text = text + '' % (snapshot['externalUrl'], '%') - for image in self.images: - if 'kpi-' in image: - if '-print' not in image: - text = text + '' % image + now = date.now() + + if title is None: + title = "%s %s %s %s:%s report" % (now.day, now.month, now.year, now.hour, now.minute) + + if grafana_dashboard is not None: + pass self.create_post(title=title, text=text, diff --git a/py-dashboard/GrafanaRequest.py b/py-dashboard/GrafanaRequest.py index f51cfbf1..fc585ab6 100644 --- a/py-dashboard/GrafanaRequest.py +++ b/py-dashboard/GrafanaRequest.py @@ -16,10 +16,10 @@ import json class GrafanaRequest: def __init__(self, + _grafana_token, _grafanajson_host, - _grafanajson_port, + grafanajson_port=3000, _folderID=0, - _api_token=None, _headers=dict(), _overwrite='false', debug_=False, @@ -27,9 +27,12 @@ class GrafanaRequest: self.debug = debug_ self.die_on_error = die_on_error_ self.headers = _headers - self.headers['Authorization'] = 'Bearer ' + _api_token + self.headers['Authorization'] = 'Bearer ' + _grafana_token self.headers['Content-Type'] = 'application/json' - self.grafanajson_url = "http://%s:%s" % (_grafanajson_host, _grafanajson_port) + self.grafanajson_host = _grafanajson_host + self.grafanajson_port = grafanajson_port + self.grafanajson_token = _grafana_token + self.grafanajson_url = "http://%s:%s" % (_grafanajson_host, grafanajson_port) self.data = dict() self.data['overwrite'] = _overwrite @@ -40,13 +43,14 @@ class GrafanaRequest: pass def list_dashboards(self): - url = self.grafanajson_url + '/api/search?folderIds=0&query=&starred=false' - return requests.get(url).text + url = self.grafanajson_url + '/api/search' + print(url) + return json.loads(requests.get(url,headers=self.headers).text) def create_dashboard(self, dashboard_name=None, ): - self.grafanajson_url = self.grafanajson_url + "/api/dashboards/db" + grafanajson_url = self.grafanajson_url + "/api/dashboards/db" datastore = dict() dashboard = dict() dashboard['id'] = None @@ -58,37 +62,59 @@ class GrafanaRequest: datastore['dashboard'] = dashboard datastore['overwrite'] = False data = json.dumps(datastore, indent=4) - return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) + return requests.post(grafanajson_url, headers=self.headers, data=data, verify=False) def delete_dashboard(self, dashboard_uid=None): - self.grafanajson_url = self.grafanajson_url + "/api/dashboards/uid/" + dashboard_uid - return requests.post(self.grafanajson_url, headers=self.headers, verify=False) + grafanajson_url = self.grafanajson_url + "/api/dashboards/uid/" + dashboard_uid + return requests.post(grafanajson_url, headers=self.headers, verify=False) def create_dashboard_from_data(self, json_file=None): - self.grafanajson_url = self.grafanajson_url + '/api/dashboards/db' + grafanajson_url = self.grafanajson_url + '/api/dashboards/db' datastore = dict() dashboard = dict(json.loads(open(json_file).read())) datastore['dashboard'] = dashboard datastore['overwrite'] = False data = json.dumps(datastore, indent=4) #return print(data) - return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) + return requests.post(grafanajson_url, headers=self.headers, data=data, verify=False) def create_dashboard_from_dict(self, dictionary=None): - self.grafanajson_url = self.grafanajson_url + '/api/dashboards/db' + grafanajson_url = self.grafanajson_url + '/api/dashboards/db' datastore = dict() dashboard = dict(json.loads(dictionary)) datastore['dashboard'] = dashboard datastore['overwrite'] = False data = json.dumps(datastore, indent=4) #return print(data) - return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) + return requests.post(grafanajson_url, headers=self.headers, data=data, verify=False) def create_custom_dashboard(self, datastore=None): data = json.dumps(datastore, indent=4) - return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) \ No newline at end of file + return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) + + def create_snapshot(self, title): + grafanajson_url = self.grafanajson_url + '/api/snapshots' + data=self.get_dashboard(title) + data['expires'] = 3600 + data['external'] = True + print(data) + return requests.post(grafanajson_url, headers=self.headers, json=data, verify=False).text + + def list_snapshots(self): + grafanajson_url = self.grafanajson_url + '/api/dashboard/snapshots' + print(grafanajson_url) + return json.loads(requests.get(grafanajson_url, headers=self.headers, verify=False).text) + + def get_dashboard(self, target): + dashboards = self.list_dashboards() + for dashboard in dashboards: + if dashboard['title'] == target: + uid = dashboard['uid'] + grafanajson_url = self.grafanajson_url + '/api/dashboards/uid/' + uid + print(grafanajson_url) + return json.loads(requests.get(grafanajson_url, headers=self.headers, verify=False).text) \ No newline at end of file diff --git a/py-scripts/ghost_profile.py b/py-scripts/ghost_profile.py index 824eab1c..2b20b377 100755 --- a/py-scripts/ghost_profile.py +++ b/py-scripts/ghost_profile.py @@ -33,7 +33,7 @@ if 'py-json' not in sys.path: from GhostRequest import GhostRequest -class UseGhost(): +class UseGhost: def __init__(self, _ghost_token=None, host="localhost", @@ -79,7 +79,12 @@ class UseGhost(): password_push, customer, testbed, - test_run): + test_run, + grafana_dashboard, + grafana_token, + grafana_host, + grafana_port): + target_folders = list() return self.GP.wifi_capacity_to_ghost(authors, folders, title, @@ -92,7 +97,12 @@ class UseGhost(): password_push, customer, testbed, - test_run) + test_run, + target_folders, + grafana_dashboard, + grafana_token, + grafana_host, + grafana_port) def main(): @@ -132,6 +142,10 @@ def main(): optional.add_argument('--customer') optional.add_argument('--testbed') optional.add_argument('--test_run', default=None) + optional.add_argument('--grafana_dashboard') + optional.add_argument('--grafana_token', default=None) + optional.add_argument('--grafana_host', default=None) + optional.add_argument('--grafana_port', default=3000) optional.add_argument('--debug') args = parser.parse_args() @@ -170,7 +184,11 @@ def main(): args.password_push, args.customer, args.testbed, - args.test_run) + args.test_run, + args.grafana_dashboard, + args.grafana_token, + args.grafana_host, + args.grafana_port) if __name__ == "__main__": diff --git a/py-scripts/grafana_profile.py b/py-scripts/grafana_profile.py index 06c15a07..a78bb987 100755 --- a/py-scripts/grafana_profile.py +++ b/py-scripts/grafana_profile.py @@ -24,36 +24,37 @@ import string import random -class UseGrafana(LFCliBase): - def __init__(self, - _grafana_token, - host="localhost", - _grafana_host="localhost", - port=8080, - _debug_on=False, - _exit_on_fail=False, - _grafana_port=3000): - super().__init__(host, port, _debug=_debug_on, _exit_on_fail=_exit_on_fail) - self.grafana_token = _grafana_token - self.grafana_port = _grafana_port - self.grafana_host = _grafana_host - self.GR = GrafanaRequest(self.grafana_host, str(self.grafana_port), _folderID=0, _api_token=self.grafana_token) +#!/usr/bin/env python3 - def create_dashboard(self, - dashboard_name): - return self.GR.create_dashboard(dashboard_name) +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Class holds default settings for json requests to Grafana - +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +import sys - def delete_dashboard(self, - dashboard_uid): - return self.GR.delete_dashboard(dashboard_uid) +if sys.version_info[0] != 3: + print("This script requires Python 3") + exit() - def list_dashboards(self): - return self.GR.list_dashboards() +import requests - def create_dashboard_from_data(self, - json_file): - return self.GR.create_dashboard_from_data(json_file=json_file) +import json +#!/usr/bin/env python3 + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Class holds default settings for json requests to Grafana - +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +import sys + +if sys.version_info[0] != 3: + print("This script requires Python 3") + exit() + +import requests + +import json + +class UseGrafana(GrafanaRequest): def groupby(self, params, grouptype): dic = dict() dic['params'] = list() @@ -301,7 +302,6 @@ class UseGrafana(LFCliBase): return dict(zip(graph_group, units)) - def main(): parser = LFCliBase.create_basic_argparse( prog='grafana_profile.py', @@ -353,11 +353,13 @@ def main(): optional.add_argument('--from_date', help='Date you want to start your Grafana dashboard from', default='now-1y') optional.add_argument('--graph_height', help='Custom height for the graph on grafana dashboard', default=8) optional.add_argument('--graph_width', help='Custom width for the graph on grafana dashboard', default=12) + optional.add_argument('--create_snapshot', action='store_true') + optional.add_argument('--list_snapshots', action='store_true') args = parser.parse_args() Grafana = UseGrafana(args.grafana_token, - args.grafana_port, - args.grafana_host + args.grafana_host, + grafanajson_port=args.grafana_port ) if args.dashboard_name is not None: Grafana.create_dashboard(args.dashboard_name) @@ -386,6 +388,13 @@ def main(): graph_height=args.graph_height, graph__width=args.graph_width) + if args.create_snapshot: + Grafana.create_snapshot(args.title) + + if args.list_snapshots: + Grafana.list_snapshots() + + if __name__ == "__main__": main()