scottk/wlan-lanforge-scripts
1
0
Fork 0
mirror of https://github.com/Telecominfraproject/wlan-lanforge-scripts.git synced 2025-10-31 18:58:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Major Ghost script improvements and additions

Browse Source 
Signed-off-by: Matthew Stidham <stidmatt@gmail.com>
This commit is contained in:
Matthew Stidham
2021-06-29 14:56:31 -07:00
parent 4dbdf2eaad
commit 8014226b32
4 changed files with 241 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
py-dashboard/GhostRequest.py
View File

@@ -61,7 +61,7 @@ class CSVReader:
def to_html(self, df): def to_html(self, df):
html = '' html = ''
html = html + ('<table style="border:1px solid #ddd"><tr>') html = html + ('<table style="border:1px solid #ddd"><tr>')
for row in df: for row in df[1:]:
for item in row: for item in row:
html = html + ('<td style="border:1px solid #ddd">%s</td>' % item) html = html + ('<td style="border:1px solid #ddd">%s</td>' % item)
html = html + ('</tr>\n<tr>') html = html + ('</tr>\n<tr>')
@@ -75,12 +75,18 @@ class CSVReader:
for row in df: for row in df:
try: try:
if expression == 'less than': if expression == 'less than':
if float(row[target_index]) <= target: if float(row[target_index]) < target:
targets.append(counter) targets.append(counter)
counter += 1 counter += 1
else: else:
counter += 1 counter += 1
if expression == 'greater than': if expression == 'greater than':
if float(row[target_index]) > target:
targets.append(counter)
counter += 1
else:
counter += 1
if expression == 'greater than or equal to':
if float(row[target_index]) >= target: if float(row[target_index]) >= target:
targets.append(counter) targets.append(counter)
counter += 1 counter += 1
@@ -90,6 +96,12 @@ class CSVReader:
counter += 1 counter += 1
return list(map(df.__getitem__, targets)) return list(map(df.__getitem__, targets))
def concat(self, dfs):
final_df = dfs[0]
for df in dfs[1:]:
final_df = final_df + df[1:]
return final_df
class GhostRequest: class GhostRequest:
def __init__(self, def __init__(self,
@@ -193,25 +205,26 @@ class GhostRequest:
tags='custom', tags='custom',
authors=authors) authors=authors)
def wifi_capacity_to_ghost(self, def kpi_to_ghost(self,
authors, authors,
folders, folders,
title=None, parent_folder=None,
server_pull=None, title=None,
ghost_host=None, server_pull=None,
port='22', ghost_host=None,
user_pull='lanforge', port=22,
password_pull='lanforge', user_push=None,
user_push=None, password_push=None,
password_push=None, customer=None,
customer=None, testbed='Unknown Testbed',
testbed='Unknown Testbed', test_run=None,
test_run=None, target_folders=list(),
target_folders=list(), grafana_dashboard=None,
grafana_dashboard=None, grafana_token=None,
grafana_token=None, grafana_host=None,
grafana_host=None, grafana_port=3000,
grafana_port=3000): grafana_datasource='InfluxDB',
grafana_bucket=None):
text = '' text = ''
csvreader = CSVReader() csvreader = CSVReader()
if grafana_token is not None: if grafana_token is not None:
@@ -219,27 +232,44 @@ class GhostRequest:
grafana_host, grafana_host,
grafanajson_port=grafana_port grafanajson_port=grafana_port
) )
if test_run is None: #if parent_folder is None:
test_run = sorted(folders)[0].split('/')[-1].strip('/') #test_run = sorted(folders)[0].split('/')[-1].strip('/')
print(folders) print('Folders: %s' % folders)
for folder in folders:
print(folder) ssh_push = paramiko.SSHClient()
ssh_pull = paramiko.SSHClient() ssh_push.set_missing_host_key_policy(paramiko.client.AutoAddPolicy)
ssh_pull.set_missing_host_key_policy(paramiko.client.AutoAddPolicy) ssh_push.connect(ghost_host,
ssh_pull.connect(server_pull, port,
port, username=user_push,
username=user_pull, password=password_push,
password=password_pull, allow_agent=False,
allow_agent=False, look_for_keys=False)
look_for_keys=False) scp_push = SCPClient(ssh_push.get_transport())
scp_pull = SCPClient(ssh_pull.get_transport())
scp_pull.get(folder, recursive=True) if parent_folder is not None:
target_folder = str(folder).rstrip('/').split('/')[-1] print("parent_folder %s" % parent_folder)
target_folders.append(target_folder) files = os.listdir(parent_folder)
print('Target folder: %s' % target_folder) print(files)
for file in files:
if os.path.isdir(parent_folder+'/'+file) is True:
import shutil
shutil.move(parent_folder+'/'+file, file)
target_folders.append(file)
print('Target folders: %s' % target_folders)
else:
for folder in folders:
print(folder)
target_folders.append(folder)
testbeds = list()
pdfs = list()
high_priority_list = list()
low_priority_list = list()
images = list()
for target_folder in target_folders:
try: try:
target_file = '%s/kpi.csv' % target_folder target_file = '%s/kpi.csv' % target_folder
print('target file %s' % target_file)
df = csvreader.read_csv(file=target_file, sep='\t') df = csvreader.read_csv(file=target_file, sep='\t')
csv_testbed = csvreader.get_column(df, 'test-rig')[0] csv_testbed = csvreader.get_column(df, 'test-rig')[0]
pass_fail = Counter(csvreader.get_column(df, 'pass/fail')) pass_fail = Counter(csvreader.get_column(df, 'pass/fail'))
@@ -249,87 +279,94 @@ class GhostRequest:
text = text + 'Percentage of tests passed: %s<br />' % ( text = text + 'Percentage of tests passed: %s<br />' % (
pass_fail['PASS'] / (pass_fail['PASS'] + pass_fail['FAIL'])) pass_fail['PASS'] / (pass_fail['PASS'] + pass_fail['FAIL']))
print(csv_testbed)
except: except:
pass print("Failure")
target_folders.remove(target_folder)
break
if len(csv_testbed) > 2: if len(csv_testbed) > 2:
testbed = csv_testbed testbed = csv_testbed
text = text + 'Testbed: %s<br />' % testbed testbeds.append(testbed)
if testbed == 'Unknown Testbed': if testbed == 'Unknown Testbed':
raise UserWarning('Please define your testbed') raise UserWarning('Please define your testbed')
print('testbed %s' % testbed)
ssh_push = paramiko.SSHClient()
ssh_push.set_missing_host_key_policy(paramiko.client.AutoAddPolicy)
ssh_push.connect(ghost_host,
port,
username=user_push,
password=password_push,
allow_agent=False,
look_for_keys=False)
scp_push = SCPClient(ssh_push.get_transport())
local_path = '/home/%s/%s/%s' % (user_push, customer, testbed) local_path = '/home/%s/%s/%s' % (user_push, customer, testbed)
transport = paramiko.Transport((ghost_host, port))
transport = paramiko.Transport(ghost_host, port)
transport.connect(None, user_push, password_push) transport.connect(None, user_push, password_push)
sftp = paramiko.sftp_client.SFTPClient.from_transport(transport) sftp = paramiko.sftp_client.SFTPClient.from_transport(transport)
print('Local Path: %s' % local_path)
try: print(local_path)
sftp.mkdir(local_path) print(target_folder)
except: scp_push.put(target_folder, local_path, recursive=True)
print('folder %s already exists' % local_path)
scp_push.put(target_folder, recursive=True, remote_path=local_path)
print(local_path + '/' + target_folder)
files = sftp.listdir(local_path + '/' + target_folder) files = sftp.listdir(local_path + '/' + target_folder)
# print('Files: %s' % files)
for file in files: for file in files:
if 'pdf' in file: if 'pdf' in file:
url = 'http://%s/%s/%s/%s/%s/%s' % ( url = 'http://%s/%s/%s/%s/%s' % (
ghost_host, customer.strip('/'), testbed, test_run, target_folder, file) ghost_host, customer.strip('/'), testbed, test_run, file)
text = text + 'PDF of results: <a href="%s">%s</a><br />' % (url, file) pdfs.append('PDF of results: <a href="%s">%s</a><br />' % (url, file))
print(url)
scp_pull.close()
scp_push.close() scp_push.close()
self.upload_images(target_folder) self.upload_images(target_folder)
for image in self.images: for image in self.images:
if 'kpi-' in image: if 'kpi-' in image:
if '-print' not in image: if '-print' not in image:
text = text + '<img src="%s"></img>' % image images.append('<img src="%s"></img>' % image)
self.images = [] self.images = []
if grafana_token is not None: results = csvreader.get_columns(df, ['short-description', 'numeric-score', 'test details', 'test-priority'])
# get the details of the dashboard through the API, and set the end date to the youngest KPI
grafana.list_dashboards()
grafana.create_snapshot(title=grafana_dashboard) low_priority = csvreader.filter_df(results, 'test-priority', 'less than', 94)
time.sleep(3) high_priority = csvreader.filter_df(results, 'test-priority', 'greater than or equal to', 95)
snapshot = grafana.list_snapshots()[-1] high_priority_list.append(high_priority)
print(snapshot)
text = text + '<iframe src="http://%s:3000/dashboard/snapshot/%s" width="100%s" height=1500></iframe><br />' % (
grafana_host, snapshot['key'], '%')
results = csvreader.get_columns(df,['short-description','numeric-score','test details','test-priority']) low_priority_list.append(low_priority)
low_priority = csvreader.to_html(csvreader.filter_df(results, 'test-priority', 'less than', 94))
high_priority = csvreader.to_html(csvreader.filter_df(results, 'test-priority', 'greater than', 95))
text = text + 'High priority results: %s' % high_priority
text = text + 'Low priority results: %s' % low_priority
now = date.now() now = date.now()
high_priority = csvreader.concat(high_priority_list)
low_priority = csvreader.concat(low_priority_list)
high_priority = csvreader.get_columns(high_priority, ['short-description', 'numeric-score', 'test details'])
low_priority = csvreader.get_columns(low_priority, ['short-description', 'numeric-score', 'test details'])
if title is None: if title is None:
title = "%s %s %s %s:%s report" % (now.day, now.month, now.year, now.hour, now.minute) title = "%s %s %s %s:%s report" % (now.day, now.month, now.year, now.hour, now.minute)
# create Grafana Dashboard # create Grafana Dashboard
target_files = [] target_files = []
for folder in folders: for folder in target_folders:
print(folder)
target_files.append(folder.split('/')[-1] + '/kpi.csv') target_files.append(folder.split('/')[-1] + '/kpi.csv')
print('Target files: %s' % target_files)
grafana.create_custom_dashboard(target_csvs=target_files, grafana.create_custom_dashboard(target_csvs=target_files,
title=title) title=title,
datasource=grafana_datasource,
bucket=grafana_bucket)
try:
text = 'Testbed: %s<br />' % testbeds[0]
except:
text = ''
for pdf in pdfs:
text = text + pdf
for image in images:
text = text + image
text = text + 'High priority results: %s' % csvreader.to_html(high_priority)
if grafana_token is not None:
# get the details of the dashboard through the API, and set the end date to the youngest KPI
grafana.list_dashboards()
grafana.create_snapshot(title='Testbed: ' + title)
time.sleep(3)
snapshot = grafana.list_snapshots()[-1]
text = text + '<iframe src="http://%s:3000/dashboard/snapshot/%s" width="100%s" height=1500></iframe><br />' % (
grafana_host, snapshot['key'], '%')
text = text + 'Low priority results: %s' % csvreader.to_html(low_priority)
self.create_post(title=title, self.create_post(title=title,
text=text, text=text,
tags='custom', tags='custom',
authors=authors) authors=authors)

69
py-dashboard/GrafanaRequest.py
View File

@@ -132,6 +132,49 @@ class GrafanaRequest:
print(dictionary) print(dictionary)
return dictionary return dictionary
def maketargets(self,
bucket,
scriptname,
groupBy,
index,
graph_group,
testbed):
query = (
'from(bucket: "%s")\n '
'|> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n '
'|> filter(fn: (r) => r["script"] == "%s")\n '
'|> group(columns: ["_measurement"])\n '
% (bucket, scriptname))
queryend = ('|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n '
'|> yield(name: "mean")\n ')
if graph_group is not None:
graphgroup = ('|> filter(fn: (r) => r["Graph-Group"] == "%s")\n' % graph_group)
query += graphgroup
if testbed is not None:
query += ('|> filter(fn: (r) => r["testbed"] == "%s")\n' % testbed)
targets = dict()
targets['delimiter'] = ','
targets['groupBy'] = groupBy
targets['header'] = True
targets['ignoreUnknown'] = False
targets['orderByTime'] = 'ASC'
targets['policy'] = 'default'
targets['query'] = query + queryend
targets['refId'] = dict(enumerate(string.ascii_uppercase, 1))[index + 1]
targets['resultFormat'] = "time_series"
targets['schema'] = list()
targets['skipRows'] = 0
targets['tags'] = list()
return targets
def groupby(self, params, grouptype):
dic = dict()
dic['params'] = list()
dic['params'].append(params)
dic['type'] = grouptype
return dic
def create_custom_dashboard(self, def create_custom_dashboard(self,
scripts=None, scripts=None,
title=None, title=None,
@@ -174,20 +217,10 @@ class GrafanaRequest:
target_csvs = open(graph_groups_file).read().split('\n') target_csvs = open(graph_groups_file).read().split('\n')
graph_groups = self.get_graph_groups( graph_groups = self.get_graph_groups(
target_csvs) # Get the list of graph groups which are in the tests we ran target_csvs) # Get the list of graph groups which are in the tests we ran
unit_dict = dict()
for csv in target_csvs:
if len(csv) > 1:
print(csv)
unit_dict.update(self.get_units(csv))
if target_csvs: if target_csvs:
print('Target CSVs: %s' % target_csvs) print('Target CSVs: %s' % target_csvs)
graph_groups = self.get_graph_groups( graph_groups = self.get_graph_groups(
target_csvs) # Get the list of graph groups which are in the tests we ran target_csvs) # Get the list of graph groups which are in the tests we ran
unit_dict = dict()
for csv in target_csvs:
if len(csv) > 1:
print(csv)
unit_dict.update(self.get_units(csv))
for scriptname in graph_groups.keys(): for scriptname in graph_groups.keys():
for graph_group in graph_groups[scriptname]: for graph_group in graph_groups[scriptname]:
panel = dict() panel = dict()
@@ -210,14 +243,14 @@ class GrafanaRequest:
options = dict() options = dict()
options['alertThreshold'] = True options['alertThreshold'] = True
#groupBy = list() groupBy = list()
#groupBy.append(self.groupby('$__interval', 'time')) groupBy.append(self.groupby('$__interval', 'time'))
#groupBy.append(self.groupby('null', 'fill')) groupBy.append(self.groupby('null', 'fill'))
#targets = list() targets = list()
#counter = 0 counter = 0
#new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group, testbed) new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group, testbed)
#targets.append(new_target) targets.append(new_target)
fieldConfig = dict() fieldConfig = dict()
fieldConfig['defaults'] = dict() fieldConfig['defaults'] = dict()
@@ -274,7 +307,7 @@ class GrafanaRequest:
panel['spaceLength'] = 10 panel['spaceLength'] = 10
panel['stack'] = False panel['stack'] = False
panel['steppedLine'] = False panel['steppedLine'] = False
#panel['targets'] = targets panel['targets'] = targets
panel['thresholds'] = list() panel['thresholds'] = list()
panel['timeFrom'] = None panel['timeFrom'] = None
panel['timeRegions'] = list() panel['timeRegions'] = list()

124
py-scripts/ghost_profile.py
View File

@@ -6,17 +6,17 @@ PURPOSE: modify ghost database from the command line.
SETUP: A Ghost installation which the user has admin access to. SETUP: A Ghost installation which the user has admin access to.
EXAMPLE: ./ghost_profile.py --article_text_file text.txt --title Test --authors Matthew --ghost_token SECRET_KEY --host 192.168.1.1 EXAMPLE: ./ghost_profile.py --article_text_file text.txt --title Test --authors Matthew --ghost_token SECRET_KEY --host 192.168.1.1
There is a specific class for uploading wifi capacity graphs called wifi_capacity. There is a specific class for uploading kpi graphs called kpi_to_ghost.
EXAMPLE: ./ghost_profile.py --ghost_token TOKEN --ghost_host 192.168.100.147 EXAMPLE: ./ghost_profile.py --ghost_token TOKEN --ghost_host 192.168.100.147
--folders /home/lanforge/html-reports/wifi-capacity-2021-06-04-02-51-07 --folders /home/lanforge/html-reports/wifi-capacity-2021-06-04-02-51-07
--wifi_capacity appl --authors Matthew --title 'wifi capacity 2021 06 04 02 51 07' --server 192.168.93.51 --kpi_to_ghost appl --authors Matthew --title 'wifi capacity 2021 06 04 02 51 07' --server 192.168.93.51
--user_pull lanforge --password_pull lanforge --customer candela --testbed heather --test_run test-run-6 --user_pull lanforge --password_pull lanforge --customer candela --testbed heather --test_run test-run-6
--user_push matt --password_push PASSWORD --user_push matt --password_push PASSWORD
EXAMPLE 2: ./ghost_profile.py --ghost_token TOKEN EXAMPLE 2: ./ghost_profile.py --ghost_token TOKEN
--ghost_host 192.168.100.147 --server 192.168.93.51 --user_pull lanforge --password_pull lanforge --customer candela --ghost_host 192.168.100.147 --server 192.168.93.51 --user_pull lanforge --password_pull lanforge --customer candela
--testbed heather --user_push matt --password_push "amount%coverage;Online" --wifi_capacity app --testbed heather --user_push matt --password_push "amount%coverage;Online" --kpi_to_ghost app
--folders /home/lanforge/html-reports/wifi-capacity-2021-06-14-10-42-29 --grafana_token TOKEN --folders /home/lanforge/html-reports/wifi-capacity-2021-06-14-10-42-29 --grafana_token TOKEN
--grafana_host 192.168.100.201 --grafana_dashboard 'Stidmatt-02' --grafana_host 192.168.100.201 --grafana_dashboard 'Stidmatt-02'
@@ -60,43 +60,45 @@ class UseGhost(GhostRequest):
text = open(file).read() text = open(file).read()
return self.create_post(title=title, text=text, tags=tags, authors=authors) return self.create_post(title=title, text=text, tags=tags, authors=authors)
def wifi_capacity(self, def kpi(self,
authors, authors,
folders, folders,
title, parent_folder,
server_pull, title,
ghost_host, server_pull,
port, ghost_host,
user_pull, port,
password_pull, user_push,
user_push, password_push,
password_push, customer,
customer, testbed,
testbed, test_run,
test_run, grafana_dashboard,
grafana_dashboard, grafana_token,
grafana_token, grafana_host,
grafana_host, grafana_port,
grafana_port): datasource,
grafana_bucket):
target_folders = list() target_folders = list()
return self.wifi_capacity_to_ghost(authors, return self.kpi_to_ghost(authors,
folders, folders,
title, parent_folder,
server_pull, title,
ghost_host, server_pull,
port, ghost_host,
user_pull, port,
password_pull, user_push,
user_push, password_push,
password_push, customer,
customer, testbed,
testbed, test_run,
test_run, target_folders,
target_folders, grafana_dashboard,
grafana_dashboard, grafana_token,
grafana_token, grafana_host,
grafana_host, grafana_port,
grafana_port) datasource,
grafana_bucket)
def main(): def main():
@@ -125,12 +127,10 @@ def main():
optional.add_argument('--image', default=None) optional.add_argument('--image', default=None)
optional.add_argument('--folder', default=None) optional.add_argument('--folder', default=None)
optional.add_argument('--custom_post', default=None) optional.add_argument('--custom_post', default=None)
optional.add_argument('--wifi_capacity', default=None) optional.add_argument('--kpi_to_ghost', help='Generate a Ghost report from KPI spreadsheets', action="store_true")
optional.add_argument('--folders', action='append', default=None) optional.add_argument('--folders', action='append', default=None)
optional.add_argument('--server_pull') optional.add_argument('--server_pull')
optional.add_argument('--port', default=22) optional.add_argument('--port', default=22)
optional.add_argument('--user_pull', default='lanforge')
optional.add_argument('--password_pull', default='lanforge')
optional.add_argument('--user_push') optional.add_argument('--user_push')
optional.add_argument('--password_push') optional.add_argument('--password_push')
optional.add_argument('--customer') optional.add_argument('--customer')
@@ -140,6 +140,9 @@ def main():
optional.add_argument('--grafana_token', default=None) optional.add_argument('--grafana_token', default=None)
optional.add_argument('--grafana_host', default=None) optional.add_argument('--grafana_host', default=None)
optional.add_argument('--grafana_port', default=3000) optional.add_argument('--grafana_port', default=3000)
optional.add_argument('--parent_folder', default=None)
optional.add_argument('--datasource', default='InfluxDB')
optional.add_argument('--grafana_bucket', default=None)
optional.add_argument('--debug') optional.add_argument('--debug')
args = parser.parse_args() args = parser.parse_args()
@@ -165,24 +168,25 @@ def main():
if args.folder is not None: if args.folder is not None:
Ghost.upload_images(args.folder) Ghost.upload_images(args.folder)
if args.wifi_capacity is not None: if args.kpi_to_ghost is True:
Ghost.wifi_capacity(args.authors, Ghost.kpi(args.authors,
args.folders, args.folders,
args.title, args.parent_folder,
args.server_pull, args.title,
args.ghost_host, args.server_pull,
args.port, args.ghost_host,
args.user_pull, args.port,
args.password_pull, args.user_push,
args.user_push, args.password_push,
args.password_push, args.customer,
args.customer, args.testbed,
args.testbed, args.test_run,
args.test_run, args.grafana_dashboard,
args.grafana_dashboard, args.grafana_token,
args.grafana_token, args.grafana_host,
args.grafana_host, args.grafana_port,
args.grafana_port) args.datasource,
args.grafana_bucket)
if __name__ == "__main__": if __name__ == "__main__":

41
py-scripts/grafana_profile.py
View File

@@ -22,47 +22,6 @@ from LANforge.lfcli_base import LFCliBase
import string import string
class UseGrafana(GrafanaRequest): class UseGrafana(GrafanaRequest):
def groupby(self, params, grouptype):
dic = dict()
dic['params'] = list()
dic['params'].append(params)
dic['type'] = grouptype
return dic
def maketargets(self,
bucket,
scriptname,
groupBy,
index,
graph_group,
testbed):
query = (
'from(bucket: "%s")\n '
'|> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n '
'|> filter(fn: (r) => r["script"] == "%s")\n '
'|> group(columns: ["_measurement"])\n '
% (bucket, scriptname))
queryend = ('|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n '
'|> yield(name: "mean")\n ')
if graph_group is not None:
graphgroup = ('|> filter(fn: (r) => r["Graph-Group"] == "%s")\n' % graph_group)
query += graphgroup
if testbed is not None:
query += ('|> filter(fn: (r) => r["testbed"] == "%s")\n' % testbed)
targets = dict()
targets['delimiter'] = ','
targets['groupBy'] = groupBy
targets['header'] = True
targets['ignoreUnknown'] = False
targets['orderByTime'] = 'ASC'
targets['policy'] = 'default'
targets['query'] = query + queryend
targets['refId'] = dict(enumerate(string.ascii_uppercase, 1))[index + 1]
targets['resultFormat'] = "time_series"
targets['schema'] = list()
targets['skipRows'] = 0
targets['tags'] = list()
return targets
def read_csv(self, file): def read_csv(self, file):