mirror of
				https://github.com/Telecominfraproject/wlan-lanforge-scripts.git
				synced 2025-10-31 02:38:03 +00:00 
			
		
		
		
	Grafana and Ghost integration updates
Signed-off-by: Matthew Stidham <stidmatt@gmail.com>
This commit is contained in:
		| @@ -20,6 +20,7 @@ import subprocess | |||||||
| from scp import SCPClient | from scp import SCPClient | ||||||
| import paramiko | import paramiko | ||||||
| from GrafanaRequest import GrafanaRequest | from GrafanaRequest import GrafanaRequest | ||||||
|  | import time | ||||||
|  |  | ||||||
|  |  | ||||||
| class CSVReader: | class CSVReader: | ||||||
| @@ -168,6 +169,11 @@ class GhostRequest: | |||||||
|                                grafana_port=3000): |                                grafana_port=3000): | ||||||
|         text = '' |         text = '' | ||||||
|         csvreader = CSVReader() |         csvreader = CSVReader() | ||||||
|  |         if grafana_token is not None: | ||||||
|  |             grafana = GrafanaRequest(grafana_token, | ||||||
|  |                                      grafana_host, | ||||||
|  |                                      grafanajson_port=grafana_port | ||||||
|  |                                      ) | ||||||
|         if test_run is None: |         if test_run is None: | ||||||
|             test_run = sorted(folders)[0].split('/')[-1].strip('/') |             test_run = sorted(folders)[0].split('/')[-1].strip('/') | ||||||
|         for folder in folders: |         for folder in folders: | ||||||
| @@ -184,7 +190,7 @@ class GhostRequest: | |||||||
|             scp_pull.get(folder, recursive=True) |             scp_pull.get(folder, recursive=True) | ||||||
|             target_folder = str(folder).rstrip('/').split('/')[-1] |             target_folder = str(folder).rstrip('/').split('/')[-1] | ||||||
|             target_folders.append(target_folder) |             target_folders.append(target_folder) | ||||||
|             print(target_folder) |             print('Target folder: %s' % target_folder) | ||||||
|             try: |             try: | ||||||
|                 target_file = '%s/kpi.csv' % target_folder |                 target_file = '%s/kpi.csv' % target_folder | ||||||
|                 print('target file %s' % target_file) |                 print('target file %s' % target_file) | ||||||
| @@ -216,9 +222,10 @@ class GhostRequest: | |||||||
|             print(local_path) |             print(local_path) | ||||||
|             try: |             try: | ||||||
|                 sftp.mkdir(local_path) |                 sftp.mkdir(local_path) | ||||||
|  |                 scp_push.put(target_folder, recursive=True, remote_path=local_path) | ||||||
|             except: |             except: | ||||||
|                 print('folder %s already exists' % local_path) |                 print('folder %s already exists' % local_path) | ||||||
|             scp_push.put(target_folder, recursive=True, remote_path=local_path) |             print(target_folder) | ||||||
|             files = sftp.listdir(local_path + '/' + target_folder) |             files = sftp.listdir(local_path + '/' + target_folder) | ||||||
|             # print('Files: %s' % files) |             # print('Files: %s' % files) | ||||||
|             for file in files: |             for file in files: | ||||||
| @@ -237,21 +244,26 @@ class GhostRequest: | |||||||
|             self.images = [] |             self.images = [] | ||||||
|  |  | ||||||
|             if grafana_token is not None: |             if grafana_token is not None: | ||||||
|                 GR = GrafanaRequest(grafana_token, |                 # get the details of the dashboard through the API, and set the end date to the youngest KPI | ||||||
|                                     grafana_host, |                 grafana.list_dashboards() | ||||||
|                                     grafanajson_port=grafana_port |  | ||||||
|                                     ) |                 grafana.create_snapshot(title=grafana_dashboard) | ||||||
|                 GR.create_snapshot(title=grafana_dashboard) |                 time.sleep(3) | ||||||
|                 snapshot = GR.list_snapshots()[-1] |                 snapshot = grafana.list_snapshots()[-1] | ||||||
|                 text = text + '<iframe src="%s" width="100%s" height=500></iframe>' % (snapshot['externalUrl'], '%') |                 print(snapshot) | ||||||
|  |                 text = text + '<iframe src="http://%s:3000/dashboard/snapshot/%s" width="100%s" height=500></iframe>' % (grafana_host, snapshot['key'], '%') | ||||||
|  |  | ||||||
|         now = date.now() |         now = date.now() | ||||||
|  |  | ||||||
|         if title is None: |         if title is None: | ||||||
|             title = "%s %s %s %s:%s report" % (now.day, now.month, now.year, now.hour, now.minute) |             title = "%s %s %s %s:%s report" % (now.day, now.month, now.year, now.hour, now.minute) | ||||||
|  |  | ||||||
|         if grafana_dashboard is not None: |         # create Grafana Dashboard | ||||||
|             pass |         target_files = [] | ||||||
|  |         for folder in folders: | ||||||
|  |             target_files.append(folder.strip('/home/lanforge/html-reports/') + '/kpi.csv') | ||||||
|  |         grafana.create_custom_dashboard(target_csvs=target_files, | ||||||
|  |                                         title=title) | ||||||
|  |  | ||||||
|         self.create_post(title=title, |         self.create_post(title=title, | ||||||
|                          text=text, |                          text=text, | ||||||
|   | |||||||
| @@ -12,7 +12,34 @@ if sys.version_info[0] != 3: | |||||||
| import requests | import requests | ||||||
|  |  | ||||||
| import json | import json | ||||||
|  | import string | ||||||
|  | import random | ||||||
|  |  | ||||||
|  | class CSVReader: | ||||||
|  |     def __init__(self): | ||||||
|  |         self.shape = None | ||||||
|  |  | ||||||
|  |     def read_csv(self, | ||||||
|  |                  file, | ||||||
|  |                  sep='\t'): | ||||||
|  |         df = open(file).read().split('\n') | ||||||
|  |         rows = list() | ||||||
|  |         for x in df: | ||||||
|  |             if len(x) > 0: | ||||||
|  |                 rows.append(x.split(sep)) | ||||||
|  |         length = list(range(0, len(df[0]))) | ||||||
|  |         columns = dict(zip(df[0], length)) | ||||||
|  |         self.shape = (length, columns) | ||||||
|  |         return rows | ||||||
|  |  | ||||||
|  |     def get_column(self, | ||||||
|  |                    df, | ||||||
|  |                    value): | ||||||
|  |         index = df[0].index(value) | ||||||
|  |         values = [] | ||||||
|  |         for row in df[1:]: | ||||||
|  |             values.append(row[index]) | ||||||
|  |         return values | ||||||
|  |  | ||||||
| class GrafanaRequest: | class GrafanaRequest: | ||||||
|     def __init__(self, |     def __init__(self, | ||||||
| @@ -35,6 +62,7 @@ class GrafanaRequest: | |||||||
|         self.grafanajson_url = "http://%s:%s" % (_grafanajson_host, grafanajson_port) |         self.grafanajson_url = "http://%s:%s" % (_grafanajson_host, grafanajson_port) | ||||||
|         self.data = dict() |         self.data = dict() | ||||||
|         self.data['overwrite'] = _overwrite |         self.data['overwrite'] = _overwrite | ||||||
|  |         self.csvreader = CSVReader() | ||||||
|  |  | ||||||
|     def create_bucket(self, |     def create_bucket(self, | ||||||
|                       bucket_name=None): |                       bucket_name=None): | ||||||
| @@ -45,7 +73,7 @@ class GrafanaRequest: | |||||||
|     def list_dashboards(self): |     def list_dashboards(self): | ||||||
|         url = self.grafanajson_url + '/api/search' |         url = self.grafanajson_url + '/api/search' | ||||||
|         print(url) |         print(url) | ||||||
|         return json.loads(requests.get(url,headers=self.headers).text) |         return json.loads(requests.get(url, headers=self.headers).text) | ||||||
|  |  | ||||||
|     def create_dashboard(self, |     def create_dashboard(self, | ||||||
|                          dashboard_name=None, |                          dashboard_name=None, | ||||||
| @@ -77,31 +105,226 @@ class GrafanaRequest: | |||||||
|         datastore['dashboard'] = dashboard |         datastore['dashboard'] = dashboard | ||||||
|         datastore['overwrite'] = False |         datastore['overwrite'] = False | ||||||
|         data = json.dumps(datastore, indent=4) |         data = json.dumps(datastore, indent=4) | ||||||
|         #return print(data) |  | ||||||
|         return requests.post(grafanajson_url, headers=self.headers, data=data, verify=False) |         return requests.post(grafanajson_url, headers=self.headers, data=data, verify=False) | ||||||
|  |  | ||||||
|     def create_dashboard_from_dict(self, |     def create_dashboard_from_dict(self, | ||||||
|                                    dictionary=None): |                                    dictionary=None, | ||||||
|  |                                    overwrite=False): | ||||||
|         grafanajson_url = self.grafanajson_url + '/api/dashboards/db' |         grafanajson_url = self.grafanajson_url + '/api/dashboards/db' | ||||||
|         datastore = dict() |         datastore = dict() | ||||||
|         dashboard = dict(json.loads(dictionary)) |         dashboard = dict(json.loads(dictionary)) | ||||||
|         datastore['dashboard'] = dashboard |         datastore['dashboard'] = dashboard | ||||||
|         datastore['overwrite'] = False |         datastore['overwrite'] = overwrite | ||||||
|         data = json.dumps(datastore, indent=4) |         data = json.dumps(datastore, indent=4) | ||||||
|         #return print(data) |  | ||||||
|         return requests.post(grafanajson_url, headers=self.headers, data=data, verify=False) |         return requests.post(grafanajson_url, headers=self.headers, data=data, verify=False) | ||||||
|  |  | ||||||
|  |     def get_graph_groups(self, target_csvs):  # Get the unique values in the Graph-Group column | ||||||
|  |         dictionary = dict() | ||||||
|  |         for target_csv in target_csvs: | ||||||
|  |             if len(target_csv) > 1: | ||||||
|  |                 csv = self.csvreader.read_csv(target_csv) | ||||||
|  |                 # Unique values in the test-id column | ||||||
|  |                 scripts = list(set(self.csvreader.get_column(csv, 'test-id'))) | ||||||
|  |                 # we need to make sure we match each Graph Group to the script it occurs in | ||||||
|  |                 for script in scripts: | ||||||
|  |                     # Unique Graph Groups for each script | ||||||
|  |                     dictionary[script] = list(set(self.csvreader.get_column(csv, 'Graph-Group'))) | ||||||
|  |         print(dictionary) | ||||||
|  |         return dictionary | ||||||
|  |  | ||||||
|     def create_custom_dashboard(self, |     def create_custom_dashboard(self, | ||||||
|                                 datastore=None): |                                 scripts=None, | ||||||
|         data = json.dumps(datastore, indent=4) |                                 title=None, | ||||||
|         return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) |                                 bucket=None, | ||||||
|  |                                 graph_groups=None, | ||||||
|  |                                 graph_groups_file=None, | ||||||
|  |                                 target_csvs=None, | ||||||
|  |                                 testbed=None, | ||||||
|  |                                 datasource='InfluxDB', | ||||||
|  |                                 from_date='now-1y', | ||||||
|  |                                 to_date='now', | ||||||
|  |                                 graph_height=8, | ||||||
|  |                                 graph__width=12): | ||||||
|  |         options = string.ascii_lowercase + string.ascii_uppercase + string.digits | ||||||
|  |         uid = ''.join(random.choice(options) for i in range(9)) | ||||||
|  |         input1 = dict() | ||||||
|  |         annotations = dict() | ||||||
|  |         annotations['builtIn'] = 1 | ||||||
|  |         annotations['datasource'] = '-- Grafana --' | ||||||
|  |         annotations['enable'] = True | ||||||
|  |         annotations['hide'] = True | ||||||
|  |         annotations['iconColor'] = 'rgba(0, 211, 255, 1)' | ||||||
|  |         annotations['name'] = 'Annotations & Alerts' | ||||||
|  |         annotations['type'] = 'dashboard' | ||||||
|  |         annot = dict() | ||||||
|  |         annot['list'] = list() | ||||||
|  |         annot['list'].append(annotations) | ||||||
|  |  | ||||||
|  |         templating = dict() | ||||||
|  |         templating['list'] = list() | ||||||
|  |  | ||||||
|  |         timedict = dict() | ||||||
|  |         timedict['from'] = from_date | ||||||
|  |         timedict['to'] = to_date | ||||||
|  |  | ||||||
|  |         panels = list() | ||||||
|  |         index = 1 | ||||||
|  |         if graph_groups_file: | ||||||
|  |             print("graph_groups_file: %s" % graph_groups_file) | ||||||
|  |             target_csvs = open(graph_groups_file).read().split('\n') | ||||||
|  |             graph_groups = self.get_graph_groups( | ||||||
|  |                 target_csvs)  # Get the list of graph groups which are in the tests we ran | ||||||
|  |             unit_dict = dict() | ||||||
|  |             for csv in target_csvs: | ||||||
|  |                 if len(csv) > 1: | ||||||
|  |                     print(csv) | ||||||
|  |                     unit_dict.update(self.get_units(csv)) | ||||||
|  |         if target_csvs: | ||||||
|  |             print('Target CSVs: %s' % target_csvs) | ||||||
|  |             graph_groups = self.get_graph_groups( | ||||||
|  |                 target_csvs)  # Get the list of graph groups which are in the tests we ran | ||||||
|  |             unit_dict = dict() | ||||||
|  |             for csv in target_csvs: | ||||||
|  |                 if len(csv) > 1: | ||||||
|  |                     print(csv) | ||||||
|  |                     unit_dict.update(self.get_units(csv)) | ||||||
|  |         for scriptname in graph_groups.keys(): | ||||||
|  |             for graph_group in graph_groups[scriptname]: | ||||||
|  |                 panel = dict() | ||||||
|  |  | ||||||
|  |                 gridpos = dict() | ||||||
|  |                 gridpos['h'] = graph_height | ||||||
|  |                 gridpos['w'] = graph__width | ||||||
|  |                 gridpos['x'] = 0 | ||||||
|  |                 gridpos['y'] = 0 | ||||||
|  |  | ||||||
|  |                 legend = dict() | ||||||
|  |                 legend['avg'] = False | ||||||
|  |                 legend['current'] = False | ||||||
|  |                 legend['max'] = False | ||||||
|  |                 legend['min'] = False | ||||||
|  |                 legend['show'] = True | ||||||
|  |                 legend['total'] = False | ||||||
|  |                 legend['values'] = False | ||||||
|  |  | ||||||
|  |                 options = dict() | ||||||
|  |                 options['alertThreshold'] = True | ||||||
|  |  | ||||||
|  |                 #groupBy = list() | ||||||
|  |                 #groupBy.append(self.groupby('$__interval', 'time')) | ||||||
|  |                 #groupBy.append(self.groupby('null', 'fill')) | ||||||
|  |  | ||||||
|  |                 #targets = list() | ||||||
|  |                 #counter = 0 | ||||||
|  |                 #new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group, testbed) | ||||||
|  |                 #targets.append(new_target) | ||||||
|  |  | ||||||
|  |                 fieldConfig = dict() | ||||||
|  |                 fieldConfig['defaults'] = dict() | ||||||
|  |                 fieldConfig['overrides'] = list() | ||||||
|  |  | ||||||
|  |                 transformation = dict() | ||||||
|  |                 transformation['id'] = "renameByRegex" | ||||||
|  |                 transformation_options = dict() | ||||||
|  |                 transformation_options['regex'] = "(.*) value.*" | ||||||
|  |                 transformation_options['renamePattern'] = "$1" | ||||||
|  |                 transformation['options'] = transformation_options | ||||||
|  |  | ||||||
|  |                 xaxis = dict() | ||||||
|  |                 xaxis['buckets'] = None | ||||||
|  |                 xaxis['mode'] = "time" | ||||||
|  |                 xaxis['name'] = None | ||||||
|  |                 xaxis['show'] = True | ||||||
|  |                 xaxis['values'] = list() | ||||||
|  |  | ||||||
|  |                 yaxis = dict() | ||||||
|  |                 yaxis['format'] = 'short' | ||||||
|  |                 #yaxis['label'] = unit_dict[graph_group] | ||||||
|  |                 yaxis['logBase'] = 1 | ||||||
|  |                 yaxis['max'] = None | ||||||
|  |                 yaxis['min'] = None | ||||||
|  |                 yaxis['show'] = True | ||||||
|  |  | ||||||
|  |                 yaxis1 = dict() | ||||||
|  |                 yaxis1['align'] = False | ||||||
|  |                 yaxis1['alignLevel'] = None | ||||||
|  |  | ||||||
|  |                 panel['aliasColors'] = dict() | ||||||
|  |                 panel['bars'] = False | ||||||
|  |                 panel['dashes'] = False | ||||||
|  |                 panel['dashLength'] = 10 | ||||||
|  |                 panel['datasource'] = datasource | ||||||
|  |                 panel['fieldConfig'] = fieldConfig | ||||||
|  |                 panel['fill'] = 0 | ||||||
|  |                 panel['fillGradient'] = 0 | ||||||
|  |                 panel['gridPos'] = gridpos | ||||||
|  |                 panel['hiddenSeries'] = False | ||||||
|  |                 panel['id'] = index | ||||||
|  |                 panel['legend'] = legend | ||||||
|  |                 panel['lines'] = True | ||||||
|  |                 panel['linewidth'] = 1 | ||||||
|  |                 panel['nullPointMode'] = 'null' | ||||||
|  |                 panel['options'] = options | ||||||
|  |                 panel['percentage'] = False | ||||||
|  |                 panel['pluginVersion'] = '7.5.4' | ||||||
|  |                 panel['pointradius'] = 2 | ||||||
|  |                 panel['points'] = True | ||||||
|  |                 panel['renderer'] = 'flot' | ||||||
|  |                 panel['seriesOverrides'] = list() | ||||||
|  |                 panel['spaceLength'] = 10 | ||||||
|  |                 panel['stack'] = False | ||||||
|  |                 panel['steppedLine'] = False | ||||||
|  |                 #panel['targets'] = targets | ||||||
|  |                 panel['thresholds'] = list() | ||||||
|  |                 panel['timeFrom'] = None | ||||||
|  |                 panel['timeRegions'] = list() | ||||||
|  |                 panel['timeShift'] = None | ||||||
|  |                 if graph_group is not None: | ||||||
|  |                     panel['title'] = scriptname + ' ' + graph_group | ||||||
|  |                 else: | ||||||
|  |                     panel['title'] = scriptname | ||||||
|  |                 panel['transformations'] = list() | ||||||
|  |                 panel['transformations'].append(transformation) | ||||||
|  |                 panel['type'] = "graph" | ||||||
|  |                 panel['xaxis'] = xaxis | ||||||
|  |                 panel['yaxes'] = list() | ||||||
|  |                 panel['yaxes'].append(yaxis) | ||||||
|  |                 panel['yaxes'].append(yaxis) | ||||||
|  |                 panel['yaxis'] = yaxis1 | ||||||
|  |  | ||||||
|  |                 panels.append(panel) | ||||||
|  |                 index = index + 1 | ||||||
|  |         input1['annotations'] = annot | ||||||
|  |         input1['editable'] = True | ||||||
|  |         input1['gnetId'] = None | ||||||
|  |         input1['graphTooltip'] = 0 | ||||||
|  |         input1['links'] = list() | ||||||
|  |         input1['panels'] = panels | ||||||
|  |         input1['refresh'] = False | ||||||
|  |         input1['schemaVersion'] = 27 | ||||||
|  |         input1['style'] = 'dark' | ||||||
|  |         input1['tags'] = list() | ||||||
|  |         input1['templating'] = templating | ||||||
|  |         input1['time'] = timedict | ||||||
|  |         input1['timepicker'] = dict() | ||||||
|  |         input1['timezone'] = '' | ||||||
|  |         input1['title'] = ("Testbed: %s" % title) | ||||||
|  |         input1['uid'] = uid | ||||||
|  |         input1['version'] = 11 | ||||||
|  |         return self.create_dashboard_from_dict(dictionary=json.dumps(input1)) | ||||||
|  |  | ||||||
|  |     #    def create_custom_dashboard(self, | ||||||
|  |     #                               datastore=None): | ||||||
|  |     #      data = json.dumps(datastore, indent=4) | ||||||
|  |     #     return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) | ||||||
|  |  | ||||||
|     def create_snapshot(self, title): |     def create_snapshot(self, title): | ||||||
|  |         print('create snapshot') | ||||||
|         grafanajson_url = self.grafanajson_url + '/api/snapshots' |         grafanajson_url = self.grafanajson_url + '/api/snapshots' | ||||||
|         data=self.get_dashboard(title) |         data = self.get_dashboard(title) | ||||||
|         data['expires'] = 3600 |         data['expires'] = 3600 | ||||||
|         data['external'] = True |         data['external'] = False | ||||||
|  |         data['timeout'] = 15 | ||||||
|         print(data) |         print(data) | ||||||
|         return requests.post(grafanajson_url, headers=self.headers, json=data, verify=False).text |         return requests.post(grafanajson_url, headers=self.headers, json=data, verify=False).text | ||||||
|  |  | ||||||
| @@ -112,9 +335,21 @@ class GrafanaRequest: | |||||||
|  |  | ||||||
|     def get_dashboard(self, target): |     def get_dashboard(self, target): | ||||||
|         dashboards = self.list_dashboards() |         dashboards = self.list_dashboards() | ||||||
|  |         print(target) | ||||||
|         for dashboard in dashboards: |         for dashboard in dashboards: | ||||||
|             if dashboard['title'] == target: |             if dashboard['title'] == target: | ||||||
|                 uid = dashboard['uid'] |                 uid = dashboard['uid'] | ||||||
|         grafanajson_url = self.grafanajson_url + '/api/dashboards/uid/' + uid |         grafanajson_url = self.grafanajson_url + '/api/dashboards/uid/' + uid | ||||||
|         print(grafanajson_url) |         print(grafanajson_url) | ||||||
|         return json.loads(requests.get(grafanajson_url, headers=self.headers, verify=False).text) |         return json.loads(requests.get(grafanajson_url, headers=self.headers, verify=False).text) | ||||||
|  |  | ||||||
|  |     def get_units(self, csv): | ||||||
|  |         df = self.csvreader.read_csv(csv) | ||||||
|  |         units = self.csvreader.get_column(df, 'Units') | ||||||
|  |         test_id = self.csvreader.get_column(df, 'test-id') | ||||||
|  |         maxunit = max(set(units), key = units.count) | ||||||
|  |         maxtest = max(set(test_id), key = test_id.count) | ||||||
|  |         d = dict() | ||||||
|  |         d[maxunit] = maxtest | ||||||
|  |         print(maxunit, maxtest) | ||||||
|  |         return d | ||||||
|   | |||||||
| @@ -229,6 +229,9 @@ class StationProfile: | |||||||
|                 self.set_command_param("add_sta", "ieee80211w", 2) |                 self.set_command_param("add_sta", "ieee80211w", 2) | ||||||
|             # self.add_sta_data["key"] = passwd |             # self.add_sta_data["key"] = passwd | ||||||
|  |  | ||||||
|  |     def station_mode_to_number(self,mode): | ||||||
|  |         modes = ['a','b','g','abg','an','abgn','bgn','bg','abgn-AC','bgn-AC','an-AC'] | ||||||
|  |  | ||||||
|     def add_security_extra(self, security): |     def add_security_extra(self, security): | ||||||
|         types = {"wep": "wep_enable", "wpa": "wpa_enable", "wpa2": "wpa2_enable", "wpa3": "use-wpa3", "open": "[BLANK]"} |         types = {"wep": "wep_enable", "wpa": "wpa_enable", "wpa2": "wpa2_enable", "wpa3": "use-wpa3", "open": "[BLANK]"} | ||||||
|         if self.desired_add_sta_flags.__contains__(types[security]) and \ |         if self.desired_add_sta_flags.__contains__(types[security]) and \ | ||||||
|   | |||||||
| @@ -14,6 +14,12 @@ EXAMPLE: ./ghost_profile.py --ghost_token TOKEN --ghost_host 192.168.100.147 | |||||||
| --user_pull lanforge --password_pull lanforge --customer candela --testbed heather --test_run test-run-6 | --user_pull lanforge --password_pull lanforge --customer candela --testbed heather --test_run test-run-6 | ||||||
| --user_push matt --password_push PASSWORD | --user_push matt --password_push PASSWORD | ||||||
|  |  | ||||||
|  | EXAMPLE 2: ./ghost_profile.py --ghost_token TOKEN | ||||||
|  | --ghost_host 192.168.100.147 --server 192.168.93.51 --user_pull lanforge --password_pull lanforge --customer candela | ||||||
|  | --testbed heather --user_push matt --password_push "amount%coverage;Online" --wifi_capacity app | ||||||
|  | --folders /home/lanforge/html-reports/wifi-capacity-2021-06-14-10-42-29 --grafana_token TOKEN | ||||||
|  | --grafana_host 192.168.100.201 --grafana_dashboard 'Stidmatt-02' | ||||||
|  |  | ||||||
|  Matthew Stidham |  Matthew Stidham | ||||||
|  Copyright 2021 Candela Technologies Inc |  Copyright 2021 Candela Technologies Inc | ||||||
|     License: Free to distribute and modify. LANforge systems must be licensed. |     License: Free to distribute and modify. LANforge systems must be licensed. | ||||||
| @@ -33,7 +39,7 @@ if 'py-json' not in sys.path: | |||||||
| from GhostRequest import GhostRequest | from GhostRequest import GhostRequest | ||||||
|  |  | ||||||
|  |  | ||||||
| class UseGhost: | class UseGhost(GhostRequest): | ||||||
|     def __init__(self, |     def __init__(self, | ||||||
|                  _ghost_token=None, |                  _ghost_token=None, | ||||||
|                  host="localhost", |                  host="localhost", | ||||||
| @@ -42,29 +48,17 @@ class UseGhost: | |||||||
|                  _exit_on_fail=False, |                  _exit_on_fail=False, | ||||||
|                  _ghost_host="localhost", |                  _ghost_host="localhost", | ||||||
|                  _ghost_port=2368, ): |                  _ghost_port=2368, ): | ||||||
|  |         super().__init__(_ghost_host, | ||||||
|  |                          str(_ghost_port), | ||||||
|  |                          _api_token=_ghost_token, | ||||||
|  |                          debug_=_debug_on) | ||||||
|         self.ghost_host = _ghost_host |         self.ghost_host = _ghost_host | ||||||
|         self.ghost_port = _ghost_port |         self.ghost_port = _ghost_port | ||||||
|         self.ghost_token = _ghost_token |         self.ghost_token = _ghost_token | ||||||
|         self.GP = GhostRequest(self.ghost_host, |  | ||||||
|                                str(self.ghost_port), |  | ||||||
|                                _api_token=self.ghost_token, |  | ||||||
|                                debug_=_debug_on) |  | ||||||
|  |  | ||||||
|     def create_post(self, title, text, tags, authors): |  | ||||||
|         return self.GP.create_post(title=title, text=text, tags=tags, authors=authors) |  | ||||||
|  |  | ||||||
|     def create_post_from_file(self, title, file, tags, authors): |     def create_post_from_file(self, title, file, tags, authors): | ||||||
|         text = open(file).read() |         text = open(file).read() | ||||||
|         return self.GP.create_post(title=title, text=text, tags=tags, authors=authors) |         return self.create_post(title=title, text=text, tags=tags, authors=authors) | ||||||
|  |  | ||||||
|     def upload_image(self, image): |  | ||||||
|         return self.GP.upload_image(image) |  | ||||||
|  |  | ||||||
|     def upload_images(self, folder): |  | ||||||
|         return self.GP.upload_images(folder) |  | ||||||
|  |  | ||||||
|     def custom_post(self, folder, authors): |  | ||||||
|         return self.GP.custom_post(folder, authors) |  | ||||||
|  |  | ||||||
|     def wifi_capacity(self, |     def wifi_capacity(self, | ||||||
|                       authors, |                       authors, | ||||||
| @@ -85,24 +79,24 @@ class UseGhost: | |||||||
|                       grafana_host, |                       grafana_host, | ||||||
|                       grafana_port): |                       grafana_port): | ||||||
|         target_folders = list() |         target_folders = list() | ||||||
|         return self.GP.wifi_capacity_to_ghost(authors, |         return self.wifi_capacity_to_ghost(authors, | ||||||
|                                               folders, |                                            folders, | ||||||
|                                               title, |                                            title, | ||||||
|                                               server_pull, |                                            server_pull, | ||||||
|                                               ghost_host, |                                            ghost_host, | ||||||
|                                               port, |                                            port, | ||||||
|                                               user_pull, |                                            user_pull, | ||||||
|                                               password_pull, |                                            password_pull, | ||||||
|                                               user_push, |                                            user_push, | ||||||
|                                               password_push, |                                            password_push, | ||||||
|                                               customer, |                                            customer, | ||||||
|                                               testbed, |                                            testbed, | ||||||
|                                               test_run, |                                            test_run, | ||||||
|                                               target_folders, |                                            target_folders, | ||||||
|                                               grafana_dashboard, |                                            grafana_dashboard, | ||||||
|                                               grafana_token, |                                            grafana_token, | ||||||
|                                               grafana_host, |                                            grafana_host, | ||||||
|                                               grafana_port) |                                            grafana_port) | ||||||
|  |  | ||||||
|  |  | ||||||
| def main(): | def main(): | ||||||
|   | |||||||
| @@ -19,40 +19,7 @@ if 'py-json' not in sys.path: | |||||||
|  |  | ||||||
| from GrafanaRequest import GrafanaRequest | from GrafanaRequest import GrafanaRequest | ||||||
| from LANforge.lfcli_base import LFCliBase | from LANforge.lfcli_base import LFCliBase | ||||||
| import json |  | ||||||
| import string | import string | ||||||
| import random |  | ||||||
|  |  | ||||||
|  |  | ||||||
| #!/usr/bin/env python3 |  | ||||||
|  |  | ||||||
| # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |  | ||||||
| # Class holds default settings for json requests to Grafana     - |  | ||||||
| # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |  | ||||||
| import sys |  | ||||||
|  |  | ||||||
| if sys.version_info[0] != 3: |  | ||||||
|     print("This script requires Python 3") |  | ||||||
|     exit() |  | ||||||
|  |  | ||||||
| import requests |  | ||||||
|  |  | ||||||
| import json |  | ||||||
|  |  | ||||||
| #!/usr/bin/env python3 |  | ||||||
|  |  | ||||||
| # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |  | ||||||
| # Class holds default settings for json requests to Grafana     - |  | ||||||
| # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |  | ||||||
| import sys |  | ||||||
|  |  | ||||||
| if sys.version_info[0] != 3: |  | ||||||
|     print("This script requires Python 3") |  | ||||||
|     exit() |  | ||||||
|  |  | ||||||
| import requests |  | ||||||
|  |  | ||||||
| import json |  | ||||||
|  |  | ||||||
| class UseGrafana(GrafanaRequest): | class UseGrafana(GrafanaRequest): | ||||||
|     def groupby(self, params, grouptype): |     def groupby(self, params, grouptype): | ||||||
| @@ -97,174 +64,6 @@ class UseGrafana(GrafanaRequest): | |||||||
|         targets['tags'] = list() |         targets['tags'] = list() | ||||||
|         return targets |         return targets | ||||||
|  |  | ||||||
|     def create_custom_dashboard(self, |  | ||||||
|                                 scripts=None, |  | ||||||
|                                 title=None, |  | ||||||
|                                 bucket=None, |  | ||||||
|                                 graph_groups=None, |  | ||||||
|                                 graph_groups_file=None, |  | ||||||
|                                 testbed=None, |  | ||||||
|                                 datasource='InfluxDB', |  | ||||||
|                                 from_date='now-1y', |  | ||||||
|                                 graph_height=8, |  | ||||||
|                                 graph__width=12): |  | ||||||
|         options = string.ascii_lowercase + string.ascii_uppercase + string.digits |  | ||||||
|         uid = ''.join(random.choice(options) for i in range(9)) |  | ||||||
|         input1 = dict() |  | ||||||
|         annotations = dict() |  | ||||||
|         annotations['builtIn'] = 1 |  | ||||||
|         annotations['datasource'] = '-- Grafana --' |  | ||||||
|         annotations['enable'] = True |  | ||||||
|         annotations['hide'] = True |  | ||||||
|         annotations['iconColor'] = 'rgba(0, 211, 255, 1)' |  | ||||||
|         annotations['name'] = 'Annotations & Alerts' |  | ||||||
|         annotations['type'] = 'dashboard' |  | ||||||
|         annot = dict() |  | ||||||
|         annot['list'] = list() |  | ||||||
|         annot['list'].append(annotations) |  | ||||||
|  |  | ||||||
|         templating = dict() |  | ||||||
|         templating['list'] = list() |  | ||||||
|  |  | ||||||
|         timedict = dict() |  | ||||||
|         timedict['from'] = from_date |  | ||||||
|         timedict['to'] = 'now' |  | ||||||
|  |  | ||||||
|         panels = list() |  | ||||||
|         index = 1 |  | ||||||
|         if graph_groups_file: |  | ||||||
|             print("graph_groups_file: %s" % graph_groups_file) |  | ||||||
|             target_csvs = open(graph_groups_file).read().split('\n') |  | ||||||
|             graph_groups = self.get_graph_groups(target_csvs)  # Get the list of graph groups which are in the tests we ran |  | ||||||
|             unit_dict = dict() |  | ||||||
|             for csv in target_csvs: |  | ||||||
|                 if len(csv)>1: |  | ||||||
|                     print(csv) |  | ||||||
|                     unit_dict.update(self.get_units(csv)) |  | ||||||
|         for scriptname in graph_groups.keys(): |  | ||||||
|             for graph_group in graph_groups[scriptname]: |  | ||||||
|                 panel = dict() |  | ||||||
|  |  | ||||||
|                 gridpos = dict() |  | ||||||
|                 gridpos['h'] = graph_height |  | ||||||
|                 gridpos['w'] = graph__width |  | ||||||
|                 gridpos['x'] = 0 |  | ||||||
|                 gridpos['y'] = 0 |  | ||||||
|  |  | ||||||
|                 legend = dict() |  | ||||||
|                 legend['avg'] = False |  | ||||||
|                 legend['current'] = False |  | ||||||
|                 legend['max'] = False |  | ||||||
|                 legend['min'] = False |  | ||||||
|                 legend['show'] = True |  | ||||||
|                 legend['total'] = False |  | ||||||
|                 legend['values'] = False |  | ||||||
|  |  | ||||||
|                 options = dict() |  | ||||||
|                 options['alertThreshold'] = True |  | ||||||
|  |  | ||||||
|                 groupBy = list() |  | ||||||
|                 groupBy.append(self.groupby('$__interval', 'time')) |  | ||||||
|                 groupBy.append(self.groupby('null', 'fill')) |  | ||||||
|  |  | ||||||
|                 targets = list() |  | ||||||
|                 counter = 0 |  | ||||||
|                 new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group,testbed) |  | ||||||
|                 targets.append(new_target) |  | ||||||
|  |  | ||||||
|                 fieldConfig = dict() |  | ||||||
|                 fieldConfig['defaults'] = dict() |  | ||||||
|                 fieldConfig['overrides'] = list() |  | ||||||
|  |  | ||||||
|                 transformation = dict() |  | ||||||
|                 transformation['id'] = "renameByRegex" |  | ||||||
|                 transformation_options = dict() |  | ||||||
|                 transformation_options['regex'] = "(.*) value.*" |  | ||||||
|                 transformation_options['renamePattern'] = "$1" |  | ||||||
|                 transformation['options'] = transformation_options |  | ||||||
|  |  | ||||||
|                 xaxis = dict() |  | ||||||
|                 xaxis['buckets'] = None |  | ||||||
|                 xaxis['mode'] = "time" |  | ||||||
|                 xaxis['name'] = None |  | ||||||
|                 xaxis['show'] = True |  | ||||||
|                 xaxis['values'] = list() |  | ||||||
|  |  | ||||||
|                 yaxis = dict() |  | ||||||
|                 yaxis['format'] = 'short' |  | ||||||
|                 yaxis['label'] = unit_dict[graph_group] |  | ||||||
|                 yaxis['logBase'] = 1 |  | ||||||
|                 yaxis['max'] = None |  | ||||||
|                 yaxis['min'] = None |  | ||||||
|                 yaxis['show'] = True |  | ||||||
|  |  | ||||||
|                 yaxis1 = dict() |  | ||||||
|                 yaxis1['align'] = False |  | ||||||
|                 yaxis1['alignLevel'] = None |  | ||||||
|  |  | ||||||
|                 panel['aliasColors'] = dict() |  | ||||||
|                 panel['bars'] = False |  | ||||||
|                 panel['dashes'] = False |  | ||||||
|                 panel['dashLength'] = 10 |  | ||||||
|                 panel['datasource'] = datasource |  | ||||||
|                 panel['fieldConfig'] = fieldConfig |  | ||||||
|                 panel['fill'] = 0 |  | ||||||
|                 panel['fillGradient'] = 0 |  | ||||||
|                 panel['gridPos'] = gridpos |  | ||||||
|                 panel['hiddenSeries'] = False |  | ||||||
|                 panel['id'] = index |  | ||||||
|                 panel['legend'] = legend |  | ||||||
|                 panel['lines'] = True |  | ||||||
|                 panel['linewidth'] = 1 |  | ||||||
|                 panel['nullPointMode'] = 'null' |  | ||||||
|                 panel['options'] = options |  | ||||||
|                 panel['percentage'] = False |  | ||||||
|                 panel['pluginVersion'] = '7.5.4' |  | ||||||
|                 panel['pointradius'] = 2 |  | ||||||
|                 panel['points'] = True |  | ||||||
|                 panel['renderer'] = 'flot' |  | ||||||
|                 panel['seriesOverrides'] = list() |  | ||||||
|                 panel['spaceLength'] = 10 |  | ||||||
|                 panel['stack'] = False |  | ||||||
|                 panel['steppedLine'] = False |  | ||||||
|                 panel['targets'] = targets |  | ||||||
|                 panel['thresholds'] = list() |  | ||||||
|                 panel['timeFrom'] = None |  | ||||||
|                 panel['timeRegions'] = list() |  | ||||||
|                 panel['timeShift'] = None |  | ||||||
|                 if graph_group is not None: |  | ||||||
|                     panel['title'] = scriptname + ' ' + graph_group |  | ||||||
|                 else: |  | ||||||
|                     panel['title'] = scriptname |  | ||||||
|                 panel['transformations'] = list() |  | ||||||
|                 panel['transformations'].append(transformation) |  | ||||||
|                 panel['type'] = "graph" |  | ||||||
|                 panel['xaxis'] = xaxis |  | ||||||
|                 panel['yaxes'] = list() |  | ||||||
|                 panel['yaxes'].append(yaxis) |  | ||||||
|                 panel['yaxes'].append(yaxis) |  | ||||||
|                 panel['yaxis'] = yaxis1 |  | ||||||
|  |  | ||||||
|                 panels.append(panel) |  | ||||||
|                 index = index + 1 |  | ||||||
|         input1['annotations'] = annot |  | ||||||
|         input1['editable'] = True |  | ||||||
|         input1['gnetId'] = None |  | ||||||
|         input1['graphTooltip'] = 0 |  | ||||||
|         input1['links'] = list() |  | ||||||
|         input1['panels'] = panels |  | ||||||
|         input1['refresh'] = False |  | ||||||
|         input1['schemaVersion'] = 27 |  | ||||||
|         input1['style'] = 'dark' |  | ||||||
|         input1['tags'] = list() |  | ||||||
|         input1['templating'] = templating |  | ||||||
|         input1['time'] = timedict |  | ||||||
|         input1['timepicker'] = dict() |  | ||||||
|         input1['timezone'] = '' |  | ||||||
|         input1['title'] = ("Testbed: %s" % title) |  | ||||||
|         input1['uid'] = uid |  | ||||||
|         input1['version'] = 11 |  | ||||||
|         return self.GR.create_dashboard_from_dict(dictionary=json.dumps(input1)) |  | ||||||
|  |  | ||||||
|     def read_csv(self, file): |     def read_csv(self, file): | ||||||
|         csv = open(file).read().split('\n') |         csv = open(file).read().split('\n') | ||||||
| @@ -281,19 +80,6 @@ class UseGrafana(GrafanaRequest): | |||||||
|             results.append(row[value]) |             results.append(row[value]) | ||||||
|         return results |         return results | ||||||
|  |  | ||||||
|     def get_graph_groups(self,target_csvs): # Get the unique values in the Graph-Group column |  | ||||||
|         dictionary = dict() |  | ||||||
|         for target_csv in target_csvs: |  | ||||||
|             if len(target_csv) > 1: |  | ||||||
|                 csv = self.read_csv(target_csv) |  | ||||||
|                 # Unique values in the test-id column |  | ||||||
|                 scripts = list(set(self.get_values(csv,'test-id'))) |  | ||||||
|                 # we need to make sure we match each Graph Group to the script it occurs in |  | ||||||
|                 for script in scripts: |  | ||||||
|                     # Unique Graph Groups for each script |  | ||||||
|                     dictionary[script] = list(set(self.get_values(csv,'Graph-Group'))) |  | ||||||
|         print(dictionary) |  | ||||||
|         return dictionary |  | ||||||
|  |  | ||||||
|     def get_units(self, target_csv): |     def get_units(self, target_csv): | ||||||
|         csv = self.read_csv(target_csv) |         csv = self.read_csv(target_csv) | ||||||
| @@ -324,6 +110,12 @@ def main(): | |||||||
|                             --graph_groups 'Per Stations Rate DL' |                             --graph_groups 'Per Stations Rate DL' | ||||||
|                             --graph_groups 'Per Stations Rate UL' |                             --graph_groups 'Per Stations Rate UL' | ||||||
|                             --graph_groups 'Per Stations Rate UL+DL' |                             --graph_groups 'Per Stations Rate UL+DL' | ||||||
|  |          | ||||||
|  |         Create a snapshot of a dashboard: | ||||||
|  |         ./grafana_profile.py --grafana_token TOKEN | ||||||
|  |                              --grafana_host HOST | ||||||
|  |                              --create_snapshot | ||||||
|  |                              --title TITLE_OF_DASHBOARD | ||||||
|             ''') |             ''') | ||||||
|     required = parser.add_argument_group('required arguments') |     required = parser.add_argument_group('required arguments') | ||||||
|     required.add_argument('--grafana_token', help='token to access your Grafana database', required=True) |     required.add_argument('--grafana_token', help='token to access your Grafana database', required=True) | ||||||
|   | |||||||
| @@ -5,11 +5,11 @@ | |||||||
| # into influxdb.  As final step, it builds a grafana dashboard for the KPI information. | # into influxdb.  As final step, it builds a grafana dashboard for the KPI information. | ||||||
|  |  | ||||||
| # Define some common variables.  This will need to be changed to match your own testbed. | # Define some common variables.  This will need to be changed to match your own testbed. | ||||||
| MGR=10.0.0.202 | MGR=192.168.93.51 | ||||||
| INFLUX_MGR=192.168.100.201 | INFLUX_MGR=192.168.100.201 | ||||||
| #INFLUXTOKEN=Tdxwq5KRbj1oNbZ_ErPL5tw_HUH2wJ1VR4dwZNugJ-APz__mEFIwnqHZdoobmQpt2fa1VdWMlHQClR8XNotwbg== | #INFLUXTOKEN=Tdxwq5KRbj1oNbZ_ErPL5tw_HUH2wJ1VR4dwZNugJ-APz__mEFIwnqHZdoobmQpt2fa1VdWMlHQClR8XNotwbg== | ||||||
| INFLUXTOKEN=31N9QDhjJHBu4eMUlMBwbK3sOjXLRAhZuCzZGeO8WVCj-xvR8gZWWvRHOcuw-5RHeB7xBFnLs7ZV023k4koR1A== | INFLUXTOKEN=31N9QDhjJHBu4eMUlMBwbK3sOjXLRAhZuCzZGeO8WVCj-xvR8gZWWvRHOcuw-5RHeB7xBFnLs7ZV023k4koR1A== | ||||||
| TESTBED=Stidmatt-01 | TESTBED=Heather | ||||||
| INFLUXBUCKET=stidmatt | INFLUXBUCKET=stidmatt | ||||||
| #GRAFANATOKEN=eyJrIjoiZTJwZkZlemhLQVNpY3hiemRjUkNBZ3k2RWc3bWpQWEkiLCJuIjoibWFzdGVyIiwiaWQiOjF9 | #GRAFANATOKEN=eyJrIjoiZTJwZkZlemhLQVNpY3hiemRjUkNBZ3k2RWc3bWpQWEkiLCJuIjoibWFzdGVyIiwiaWQiOjF9 | ||||||
| GRAFANATOKEN=eyJrIjoiS1NGRU8xcTVBQW9lUmlTM2dNRFpqNjFqV05MZkM0dzciLCJuIjoibWF0dGhldyIsImlkIjoxfQ== | GRAFANATOKEN=eyJrIjoiS1NGRU8xcTVBQW9lUmlTM2dNRFpqNjFqV05MZkM0dzciLCJuIjoibWF0dGhldyIsImlkIjoxfQ== | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user
	 Matthew Stidham
					Matthew Stidham