scottk/wlan-lanforge-scripts
1
0
Fork 0
mirror of https://github.com/Telecominfraproject/wlan-lanforge-scripts.git synced 2025-10-31 02:38:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Grafana and Ghost integration updates

Browse Source 
Signed-off-by: Matthew Stidham <stidmatt@gmail.com>
This commit is contained in:
Matthew Stidham
2021-06-22 14:30:38 -07:00
parent 5f947ec16c
commit c2586f8343
6 changed files with 310 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
py-dashboard/GhostRequest.py
View File

@@ -20,6 +20,7 @@ import subprocess
from scp import SCPClient
import paramiko
from GrafanaRequest import GrafanaRequest
import time
class CSVReader:
@@ -168,6 +169,11 @@ class GhostRequest:
grafana_port=3000):
text = ''
csvreader = CSVReader()
if grafana_token is not None:
grafana = GrafanaRequest(grafana_token,
grafana_host,
grafanajson_port=grafana_port
)
if test_run is None:
test_run = sorted(folders)[0].split('/')[-1].strip('/')
for folder in folders:
@@ -184,7 +190,7 @@ class GhostRequest:
scp_pull.get(folder, recursive=True)
target_folder = str(folder).rstrip('/').split('/')[-1]
target_folders.append(target_folder)
print(target_folder)
print('Target folder: %s' % target_folder)
try:
target_file = '%s/kpi.csv' % target_folder
print('target file %s' % target_file)
@@ -216,9 +222,10 @@ class GhostRequest:
print(local_path)
try:
sftp.mkdir(local_path)
scp_push.put(target_folder, recursive=True, remote_path=local_path)
except:
print('folder %s already exists' % local_path)
scp_push.put(target_folder, recursive=True, remote_path=local_path)
print(target_folder)
files = sftp.listdir(local_path + '/' + target_folder)
# print('Files: %s' % files)
for file in files:
@@ -237,21 +244,26 @@ class GhostRequest:
self.images = []
if grafana_token is not None:
GR = GrafanaRequest(grafana_token,
grafana_host,
grafanajson_port=grafana_port
)
GR.create_snapshot(title=grafana_dashboard)
snapshot = GR.list_snapshots()[-1]
text = text + '<iframe src="%s" width="100%s" height=500></iframe>' % (snapshot['externalUrl'], '%')
# get the details of the dashboard through the API, and set the end date to the youngest KPI
grafana.list_dashboards()
grafana.create_snapshot(title=grafana_dashboard)
time.sleep(3)
snapshot = grafana.list_snapshots()[-1]
print(snapshot)
text = text + '<iframe src="http://%s:3000/dashboard/snapshot/%s" width="100%s" height=500></iframe>' % (grafana_host, snapshot['key'], '%')
now = date.now()
if title is None:
title = "%s %s %s %s:%s report" % (now.day, now.month, now.year, now.hour, now.minute)
if grafana_dashboard is not None:
pass
# create Grafana Dashboard
target_files = []
for folder in folders:
target_files.append(folder.strip('/home/lanforge/html-reports/') + '/kpi.csv')
grafana.create_custom_dashboard(target_csvs=target_files,
title=title)
self.create_post(title=title,
text=text,

255
py-dashboard/GrafanaRequest.py
View File

@@ -12,7 +12,34 @@ if sys.version_info[0] != 3:
import requests
import json
import string
import random
class CSVReader:
def __init__(self):
self.shape = None
def read_csv(self,
file,
sep='\t'):
df = open(file).read().split('\n')
rows = list()
for x in df:
if len(x) > 0:
rows.append(x.split(sep))
length = list(range(0, len(df[0])))
columns = dict(zip(df[0], length))
self.shape = (length, columns)
return rows
def get_column(self,
df,
value):
index = df[0].index(value)
values = []
for row in df[1:]:
values.append(row[index])
return values
class GrafanaRequest:
def __init__(self,
@@ -35,6 +62,7 @@ class GrafanaRequest:
self.grafanajson_url = "http://%s:%s" % (_grafanajson_host, grafanajson_port)
self.data = dict()
self.data['overwrite'] = _overwrite
self.csvreader = CSVReader()
def create_bucket(self,
bucket_name=None):
@@ -45,7 +73,7 @@ class GrafanaRequest:
def list_dashboards(self):
url = self.grafanajson_url + '/api/search'
print(url)
return json.loads(requests.get(url,headers=self.headers).text)
return json.loads(requests.get(url, headers=self.headers).text)
def create_dashboard(self,
dashboard_name=None,
@@ -77,31 +105,226 @@ class GrafanaRequest:
datastore['dashboard'] = dashboard
datastore['overwrite'] = False
data = json.dumps(datastore, indent=4)
#return print(data)
return requests.post(grafanajson_url, headers=self.headers, data=data, verify=False)
def create_dashboard_from_dict(self,
dictionary=None):
dictionary=None,
overwrite=False):
grafanajson_url = self.grafanajson_url + '/api/dashboards/db'
datastore = dict()
dashboard = dict(json.loads(dictionary))
datastore['dashboard'] = dashboard
datastore['overwrite'] = False
datastore['overwrite'] = overwrite
data = json.dumps(datastore, indent=4)
#return print(data)
return requests.post(grafanajson_url, headers=self.headers, data=data, verify=False)
def get_graph_groups(self, target_csvs): # Get the unique values in the Graph-Group column
dictionary = dict()
for target_csv in target_csvs:
if len(target_csv) > 1:
csv = self.csvreader.read_csv(target_csv)
# Unique values in the test-id column
scripts = list(set(self.csvreader.get_column(csv, 'test-id')))
# we need to make sure we match each Graph Group to the script it occurs in
for script in scripts:
# Unique Graph Groups for each script
dictionary[script] = list(set(self.csvreader.get_column(csv, 'Graph-Group')))
print(dictionary)
return dictionary
def create_custom_dashboard(self,
datastore=None):
data = json.dumps(datastore, indent=4)
return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False)
scripts=None,
title=None,
bucket=None,
graph_groups=None,
graph_groups_file=None,
target_csvs=None,
testbed=None,
datasource='InfluxDB',
from_date='now-1y',
to_date='now',
graph_height=8,
graph__width=12):
options = string.ascii_lowercase + string.ascii_uppercase + string.digits
uid = ''.join(random.choice(options) for i in range(9))
input1 = dict()
annotations = dict()
annotations['builtIn'] = 1
annotations['datasource'] = '-- Grafana --'
annotations['enable'] = True
annotations['hide'] = True
annotations['iconColor'] = 'rgba(0, 211, 255, 1)'
annotations['name'] = 'Annotations & Alerts'
annotations['type'] = 'dashboard'
annot = dict()
annot['list'] = list()
annot['list'].append(annotations)
templating = dict()
templating['list'] = list()
timedict = dict()
timedict['from'] = from_date
timedict['to'] = to_date
panels = list()
index = 1
if graph_groups_file:
print("graph_groups_file: %s" % graph_groups_file)
target_csvs = open(graph_groups_file).read().split('\n')
graph_groups = self.get_graph_groups(
target_csvs) # Get the list of graph groups which are in the tests we ran
unit_dict = dict()
for csv in target_csvs:
if len(csv) > 1:
print(csv)
unit_dict.update(self.get_units(csv))
if target_csvs:
print('Target CSVs: %s' % target_csvs)
graph_groups = self.get_graph_groups(
target_csvs) # Get the list of graph groups which are in the tests we ran
unit_dict = dict()
for csv in target_csvs:
if len(csv) > 1:
print(csv)
unit_dict.update(self.get_units(csv))
for scriptname in graph_groups.keys():
for graph_group in graph_groups[scriptname]:
panel = dict()
gridpos = dict()
gridpos['h'] = graph_height
gridpos['w'] = graph__width
gridpos['x'] = 0
gridpos['y'] = 0
legend = dict()
legend['avg'] = False
legend['current'] = False
legend['max'] = False
legend['min'] = False
legend['show'] = True
legend['total'] = False
legend['values'] = False
options = dict()
options['alertThreshold'] = True
#groupBy = list()
#groupBy.append(self.groupby('$__interval', 'time'))
#groupBy.append(self.groupby('null', 'fill'))
#targets = list()
#counter = 0
#new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group, testbed)
#targets.append(new_target)
fieldConfig = dict()
fieldConfig['defaults'] = dict()
fieldConfig['overrides'] = list()
transformation = dict()
transformation['id'] = "renameByRegex"
transformation_options = dict()
transformation_options['regex'] = "(.*) value.*"
transformation_options['renamePattern'] = "$1"
transformation['options'] = transformation_options
xaxis = dict()
xaxis['buckets'] = None
xaxis['mode'] = "time"
xaxis['name'] = None
xaxis['show'] = True
xaxis['values'] = list()
yaxis = dict()
yaxis['format'] = 'short'
#yaxis['label'] = unit_dict[graph_group]
yaxis['logBase'] = 1
yaxis['max'] = None
yaxis['min'] = None
yaxis['show'] = True
yaxis1 = dict()
yaxis1['align'] = False
yaxis1['alignLevel'] = None
panel['aliasColors'] = dict()
panel['bars'] = False
panel['dashes'] = False
panel['dashLength'] = 10
panel['datasource'] = datasource
panel['fieldConfig'] = fieldConfig
panel['fill'] = 0
panel['fillGradient'] = 0
panel['gridPos'] = gridpos
panel['hiddenSeries'] = False
panel['id'] = index
panel['legend'] = legend
panel['lines'] = True
panel['linewidth'] = 1
panel['nullPointMode'] = 'null'
panel['options'] = options
panel['percentage'] = False
panel['pluginVersion'] = '7.5.4'
panel['pointradius'] = 2
panel['points'] = True
panel['renderer'] = 'flot'
panel['seriesOverrides'] = list()
panel['spaceLength'] = 10
panel['stack'] = False
panel['steppedLine'] = False
#panel['targets'] = targets
panel['thresholds'] = list()
panel['timeFrom'] = None
panel['timeRegions'] = list()
panel['timeShift'] = None
if graph_group is not None:
panel['title'] = scriptname + ' ' + graph_group
else:
panel['title'] = scriptname
panel['transformations'] = list()
panel['transformations'].append(transformation)
panel['type'] = "graph"
panel['xaxis'] = xaxis
panel['yaxes'] = list()
panel['yaxes'].append(yaxis)
panel['yaxes'].append(yaxis)
panel['yaxis'] = yaxis1
panels.append(panel)
index = index + 1
input1['annotations'] = annot
input1['editable'] = True
input1['gnetId'] = None
input1['graphTooltip'] = 0
input1['links'] = list()
input1['panels'] = panels
input1['refresh'] = False
input1['schemaVersion'] = 27
input1['style'] = 'dark'
input1['tags'] = list()
input1['templating'] = templating
input1['time'] = timedict
input1['timepicker'] = dict()
input1['timezone'] = ''
input1['title'] = ("Testbed: %s" % title)
input1['uid'] = uid
input1['version'] = 11
return self.create_dashboard_from_dict(dictionary=json.dumps(input1))
# def create_custom_dashboard(self,
# datastore=None):
# data = json.dumps(datastore, indent=4)
# return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False)
def create_snapshot(self, title):
print('create snapshot')
grafanajson_url = self.grafanajson_url + '/api/snapshots'
data=self.get_dashboard(title)
data = self.get_dashboard(title)
data['expires'] = 3600
data['external'] = True
data['external'] = False
data['timeout'] = 15
print(data)
return requests.post(grafanajson_url, headers=self.headers, json=data, verify=False).text
@@ -112,9 +335,21 @@ class GrafanaRequest:
def get_dashboard(self, target):
dashboards = self.list_dashboards()
print(target)
for dashboard in dashboards:
if dashboard['title'] == target:
uid = dashboard['uid']
grafanajson_url = self.grafanajson_url + '/api/dashboards/uid/' + uid
print(grafanajson_url)
return json.loads(requests.get(grafanajson_url, headers=self.headers, verify=False).text)
def get_units(self, csv):
df = self.csvreader.read_csv(csv)
units = self.csvreader.get_column(df, 'Units')
test_id = self.csvreader.get_column(df, 'test-id')
maxunit = max(set(units), key = units.count)
maxtest = max(set(test_id), key = test_id.count)
d = dict()
d[maxunit] = maxtest
print(maxunit, maxtest)
return d

3
py-json/station_profile.py
View File

@@ -229,6 +229,9 @@ class StationProfile:
self.set_command_param("add_sta", "ieee80211w", 2)
# self.add_sta_data["key"] = passwd
def station_mode_to_number(self,mode):
modes = ['a','b','g','abg','an','abgn','bgn','bg','abgn-AC','bgn-AC','an-AC']
def add_security_extra(self, security):
types = {"wep": "wep_enable", "wpa": "wpa_enable", "wpa2": "wpa2_enable", "wpa3": "use-wpa3", "open": "[BLANK]"}
if self.desired_add_sta_flags.__contains__(types[security]) and \

32
py-scripts/ghost_profile.py
View File

@@ -14,6 +14,12 @@ EXAMPLE: ./ghost_profile.py --ghost_token TOKEN --ghost_host 192.168.100.147
--user_pull lanforge --password_pull lanforge --customer candela --testbed heather --test_run test-run-6
--user_push matt --password_push PASSWORD
EXAMPLE 2: ./ghost_profile.py --ghost_token TOKEN
--ghost_host 192.168.100.147 --server 192.168.93.51 --user_pull lanforge --password_pull lanforge --customer candela
--testbed heather --user_push matt --password_push "amount%coverage;Online" --wifi_capacity app
--folders /home/lanforge/html-reports/wifi-capacity-2021-06-14-10-42-29 --grafana_token TOKEN
--grafana_host 192.168.100.201 --grafana_dashboard 'Stidmatt-02'
Matthew Stidham
Copyright 2021 Candela Technologies Inc
License: Free to distribute and modify. LANforge systems must be licensed.
@@ -33,7 +39,7 @@ if 'py-json' not in sys.path:
from GhostRequest import GhostRequest
class UseGhost:
class UseGhost(GhostRequest):
def __init__(self,
_ghost_token=None,
host="localhost",
@@ -42,29 +48,17 @@ class UseGhost:
_exit_on_fail=False,
_ghost_host="localhost",
_ghost_port=2368, ):
super().__init__(_ghost_host,
str(_ghost_port),
_api_token=_ghost_token,
debug_=_debug_on)
self.ghost_host = _ghost_host
self.ghost_port = _ghost_port
self.ghost_token = _ghost_token
self.GP = GhostRequest(self.ghost_host,
str(self.ghost_port),
_api_token=self.ghost_token,
debug_=_debug_on)
def create_post(self, title, text, tags, authors):
return self.GP.create_post(title=title, text=text, tags=tags, authors=authors)
def create_post_from_file(self, title, file, tags, authors):
text = open(file).read()
return self.GP.create_post(title=title, text=text, tags=tags, authors=authors)
def upload_image(self, image):
return self.GP.upload_image(image)
def upload_images(self, folder):
return self.GP.upload_images(folder)
def custom_post(self, folder, authors):
return self.GP.custom_post(folder, authors)
return self.create_post(title=title, text=text, tags=tags, authors=authors)
def wifi_capacity(self,
authors,
@@ -85,7 +79,7 @@ class UseGhost:
grafana_host,
grafana_port):
target_folders = list()
return self.GP.wifi_capacity_to_ghost(authors,
return self.wifi_capacity_to_ghost(authors,
folders,
title,
server_pull,

220
py-scripts/grafana_profile.py
View File

@@ -19,40 +19,7 @@ if 'py-json' not in sys.path:
from GrafanaRequest import GrafanaRequest
from LANforge.lfcli_base import LFCliBase
import json
import string
import random
#!/usr/bin/env python3
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Class holds default settings for json requests to Grafana -
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
import sys
if sys.version_info[0] != 3:
print("This script requires Python 3")
exit()
import requests
import json
#!/usr/bin/env python3
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Class holds default settings for json requests to Grafana -
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
import sys
if sys.version_info[0] != 3:
print("This script requires Python 3")
exit()
import requests
import json
class UseGrafana(GrafanaRequest):
def groupby(self, params, grouptype):
@@ -97,174 +64,6 @@ class UseGrafana(GrafanaRequest):
targets['tags'] = list()
return targets
def create_custom_dashboard(self,
scripts=None,
title=None,
bucket=None,
graph_groups=None,
graph_groups_file=None,
testbed=None,
datasource='InfluxDB',
from_date='now-1y',
graph_height=8,
graph__width=12):
options = string.ascii_lowercase + string.ascii_uppercase + string.digits
uid = ''.join(random.choice(options) for i in range(9))
input1 = dict()
annotations = dict()
annotations['builtIn'] = 1
annotations['datasource'] = '-- Grafana --'
annotations['enable'] = True
annotations['hide'] = True
annotations['iconColor'] = 'rgba(0, 211, 255, 1)'
annotations['name'] = 'Annotations & Alerts'
annotations['type'] = 'dashboard'
annot = dict()
annot['list'] = list()
annot['list'].append(annotations)
templating = dict()
templating['list'] = list()
timedict = dict()
timedict['from'] = from_date
timedict['to'] = 'now'
panels = list()
index = 1
if graph_groups_file:
print("graph_groups_file: %s" % graph_groups_file)
target_csvs = open(graph_groups_file).read().split('\n')
graph_groups = self.get_graph_groups(target_csvs) # Get the list of graph groups which are in the tests we ran
unit_dict = dict()
for csv in target_csvs:
if len(csv)>1:
print(csv)
unit_dict.update(self.get_units(csv))
for scriptname in graph_groups.keys():
for graph_group in graph_groups[scriptname]:
panel = dict()
gridpos = dict()
gridpos['h'] = graph_height
gridpos['w'] = graph__width
gridpos['x'] = 0
gridpos['y'] = 0
legend = dict()
legend['avg'] = False
legend['current'] = False
legend['max'] = False
legend['min'] = False
legend['show'] = True
legend['total'] = False
legend['values'] = False
options = dict()
options['alertThreshold'] = True
groupBy = list()
groupBy.append(self.groupby('$__interval', 'time'))
groupBy.append(self.groupby('null', 'fill'))
targets = list()
counter = 0
new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group,testbed)
targets.append(new_target)
fieldConfig = dict()
fieldConfig['defaults'] = dict()
fieldConfig['overrides'] = list()
transformation = dict()
transformation['id'] = "renameByRegex"
transformation_options = dict()
transformation_options['regex'] = "(.*) value.*"
transformation_options['renamePattern'] = "$1"
transformation['options'] = transformation_options
xaxis = dict()
xaxis['buckets'] = None
xaxis['mode'] = "time"
xaxis['name'] = None
xaxis['show'] = True
xaxis['values'] = list()
yaxis = dict()
yaxis['format'] = 'short'
yaxis['label'] = unit_dict[graph_group]
yaxis['logBase'] = 1
yaxis['max'] = None
yaxis['min'] = None
yaxis['show'] = True
yaxis1 = dict()
yaxis1['align'] = False
yaxis1['alignLevel'] = None
panel['aliasColors'] = dict()
panel['bars'] = False
panel['dashes'] = False
panel['dashLength'] = 10
panel['datasource'] = datasource
panel['fieldConfig'] = fieldConfig
panel['fill'] = 0
panel['fillGradient'] = 0
panel['gridPos'] = gridpos
panel['hiddenSeries'] = False
panel['id'] = index
panel['legend'] = legend
panel['lines'] = True
panel['linewidth'] = 1
panel['nullPointMode'] = 'null'
panel['options'] = options
panel['percentage'] = False
panel['pluginVersion'] = '7.5.4'
panel['pointradius'] = 2
panel['points'] = True
panel['renderer'] = 'flot'
panel['seriesOverrides'] = list()
panel['spaceLength'] = 10
panel['stack'] = False
panel['steppedLine'] = False
panel['targets'] = targets
panel['thresholds'] = list()
panel['timeFrom'] = None
panel['timeRegions'] = list()
panel['timeShift'] = None
if graph_group is not None:
panel['title'] = scriptname + ' ' + graph_group
else:
panel['title'] = scriptname
panel['transformations'] = list()
panel['transformations'].append(transformation)
panel['type'] = "graph"
panel['xaxis'] = xaxis
panel['yaxes'] = list()
panel['yaxes'].append(yaxis)
panel['yaxes'].append(yaxis)
panel['yaxis'] = yaxis1
panels.append(panel)
index = index + 1
input1['annotations'] = annot
input1['editable'] = True
input1['gnetId'] = None
input1['graphTooltip'] = 0
input1['links'] = list()
input1['panels'] = panels
input1['refresh'] = False
input1['schemaVersion'] = 27
input1['style'] = 'dark'
input1['tags'] = list()
input1['templating'] = templating
input1['time'] = timedict
input1['timepicker'] = dict()
input1['timezone'] = ''
input1['title'] = ("Testbed: %s" % title)
input1['uid'] = uid
input1['version'] = 11
return self.GR.create_dashboard_from_dict(dictionary=json.dumps(input1))
def read_csv(self, file):
csv = open(file).read().split('\n')
@@ -281,19 +80,6 @@ class UseGrafana(GrafanaRequest):
results.append(row[value])
return results
def get_graph_groups(self,target_csvs): # Get the unique values in the Graph-Group column
dictionary = dict()
for target_csv in target_csvs:
if len(target_csv) > 1:
csv = self.read_csv(target_csv)
# Unique values in the test-id column
scripts = list(set(self.get_values(csv,'test-id')))
# we need to make sure we match each Graph Group to the script it occurs in
for script in scripts:
# Unique Graph Groups for each script
dictionary[script] = list(set(self.get_values(csv,'Graph-Group')))
print(dictionary)
return dictionary
def get_units(self, target_csv):
csv = self.read_csv(target_csv)
@@ -324,6 +110,12 @@ def main():
--graph_groups 'Per Stations Rate DL'
--graph_groups 'Per Stations Rate UL'
--graph_groups 'Per Stations Rate UL+DL'
Create a snapshot of a dashboard:
./grafana_profile.py --grafana_token TOKEN
--grafana_host HOST
--create_snapshot
--title TITLE_OF_DASHBOARD
''')
required = parser.add_argument_group('required arguments')
required.add_argument('--grafana_token', help='token to access your Grafana database', required=True)

4
py-scripts/wifi_cap_to_grafana.sh
View File

@@ -5,11 +5,11 @@
# into influxdb. As final step, it builds a grafana dashboard for the KPI information.
# Define some common variables. This will need to be changed to match your own testbed.
MGR=10.0.0.202
MGR=192.168.93.51
INFLUX_MGR=192.168.100.201
#INFLUXTOKEN=Tdxwq5KRbj1oNbZ_ErPL5tw_HUH2wJ1VR4dwZNugJ-APz__mEFIwnqHZdoobmQpt2fa1VdWMlHQClR8XNotwbg==
INFLUXTOKEN=31N9QDhjJHBu4eMUlMBwbK3sOjXLRAhZuCzZGeO8WVCj-xvR8gZWWvRHOcuw-5RHeB7xBFnLs7ZV023k4koR1A==
TESTBED=Stidmatt-01
TESTBED=Heather
INFLUXBUCKET=stidmatt
#GRAFANATOKEN=eyJrIjoiZTJwZkZlemhLQVNpY3hiemRjUkNBZ3k2RWc3bWpQWEkiLCJuIjoibWFzdGVyIiwiaWQiOjF9
GRAFANATOKEN=eyJrIjoiS1NGRU8xcTVBQW9lUmlTM2dNRFpqNjFqV05MZkM0dzciLCJuIjoibWF0dGhldyIsImlkIjoxfQ==