mirror of
https://github.com/Telecominfraproject/wlan-lanforge-scripts.git
synced 2025-11-03 20:27:54 +00:00
Make Ghost Request more versatile as kpi.csv files change so it doesn't break on older lanforge builds, and so we can add in new columns without difficulty.
Signed-off-by: Matthew Stidham <stidmatt@gmail.com>
This commit is contained in:
Matthew Stidham
@@ -15,7 +15,6 @@ import requests
|
||||
|
||||
import jwt
|
||||
from datetime import datetime
|
||||
from dateutil import tz
|
||||
import json
|
||||
import subprocess
|
||||
from scp import SCPClient
|
||||
@@ -25,6 +24,7 @@ from influx2 import RecordInflux
|
||||
import time
|
||||
from collections import Counter
|
||||
import shutil
|
||||
import itertools
|
||||
|
||||
|
||||
class CSVReader:
|
||||
@@ -36,8 +36,6 @@ class CSVReader:
|
||||
for x in df:
|
||||
if len(x) > 0:
|
||||
rows.append(x.split(sep))
|
||||
length = list(range(0, len(df[0])))
|
||||
columns = dict(zip(df[0], length))
|
||||
return rows
|
||||
|
||||
def get_column(self,
|
||||
@@ -74,7 +72,7 @@ class CSVReader:
|
||||
for row in df:
|
||||
for item in row:
|
||||
html = html + ('<td style="border:1px solid #ddd">%s</td>' % item)
|
||||
html = html + ('</tr>\n<tr>')
|
||||
html = html + '</tr>\n<tr>'
|
||||
html = html + ('</tbody>'
|
||||
'</table>')
|
||||
return html
|
||||
@@ -83,35 +81,24 @@ class CSVReader:
|
||||
target_index = df[0].index(column)
|
||||
counter = 0
|
||||
targets = [0]
|
||||
for row in df:
|
||||
for row in df[1:]:
|
||||
try:
|
||||
if expression == 'less than':
|
||||
if float(row[target_index]) < target:
|
||||
targets.append(counter)
|
||||
counter += 1
|
||||
else:
|
||||
counter += 1
|
||||
if expression == 'greater than':
|
||||
if float(row[target_index]) > target:
|
||||
targets.append(counter)
|
||||
counter += 1
|
||||
else:
|
||||
counter += 1
|
||||
if expression == 'greater than or equal to':
|
||||
if float(row[target_index]) >= target:
|
||||
targets.append(counter)
|
||||
counter += 1
|
||||
else:
|
||||
counter += 1
|
||||
except:
|
||||
counter += 1
|
||||
finally:
|
||||
pass
|
||||
counter += 1
|
||||
return list(map(df.__getitem__, targets))
|
||||
|
||||
def concat(self, dfs):
|
||||
final_df = dfs[0]
|
||||
for df in dfs[1:]:
|
||||
final_df = final_df + df[1:]
|
||||
return final_df
|
||||
return list(itertools.chain.from_iterable(dfs))
|
||||
|
||||
|
||||
class GhostRequest:
|
||||
@@ -165,8 +152,6 @@ class GhostRequest:
|
||||
def create_post(self,
|
||||
title=None,
|
||||
text=None,
|
||||
tags=None,
|
||||
authors=None,
|
||||
status="published"):
|
||||
ghost_json_url = self.ghost_json_url + '/admin/posts/?source=html'
|
||||
post = dict()
|
||||
@@ -199,9 +184,9 @@ class GhostRequest:
|
||||
ghost_json_url = self.ghost_json_url + '/admin/images/upload/'
|
||||
|
||||
token = self.encode_token()
|
||||
bashCommand = "curl -X POST -F 'file=@%s' -H \"Authorization: Ghost %s\" %s" % (image, token, ghost_json_url)
|
||||
bash_command = "curl -X POST -F 'file=@%s' -H \"Authorization: Ghost %s\" %s" % (image, token, ghost_json_url)
|
||||
|
||||
proc = subprocess.Popen(bashCommand, shell=True, stdout=subprocess.PIPE)
|
||||
proc = subprocess.Popen(bash_command, shell=True, stdout=subprocess.PIPE)
|
||||
output = proc.stdout.read().decode('utf-8')
|
||||
if self.debug:
|
||||
print(output)
|
||||
@@ -226,12 +211,7 @@ class GhostRequest:
|
||||
head = head + '<img src="%s"></img>' % picture
|
||||
head = head + '''<p>This is the end of the example</p>'''
|
||||
self.create_post(title=title,
|
||||
text=head,
|
||||
tags='custom',
|
||||
authors=authors)
|
||||
|
||||
def list_append(self, list_1, value):
|
||||
list_1.append(value)
|
||||
text=head)
|
||||
|
||||
def kpi_to_ghost(self,
|
||||
authors,
|
||||
@@ -258,11 +238,6 @@ class GhostRequest:
|
||||
|
||||
text = ''
|
||||
csvreader = CSVReader()
|
||||
if grafana_token is not None:
|
||||
grafana = GrafanaRequest(grafana_token,
|
||||
grafana_host,
|
||||
grafanajson_port=grafana_port
|
||||
)
|
||||
if self.debug:
|
||||
print('Folders: %s' % folders)
|
||||
|
||||
@@ -296,7 +271,7 @@ class GhostRequest:
|
||||
target_folders.append(folder)
|
||||
|
||||
testbeds = list()
|
||||
webpagesandpdfs = list()
|
||||
web_pages_and_pdfs = list()
|
||||
high_priority_list = list()
|
||||
low_priority_list = list()
|
||||
images = list()
|
||||
@@ -306,23 +281,27 @@ class GhostRequest:
|
||||
subtest_pass_total = 0
|
||||
subtest_fail_total = 0
|
||||
test_tag = dict()
|
||||
columns = ['test-rig', 'dut-hw-version', 'dut-sw-version',
|
||||
'dut-model-num', 'dut-serial-num']
|
||||
duts = dict()
|
||||
|
||||
for target_folder in target_folders:
|
||||
try:
|
||||
target_file = '%s/kpi.csv' % target_folder
|
||||
df = csvreader.read_csv(file=target_file, sep='\t')
|
||||
test_rig = csvreader.get_column(df, 'test-rig')[0]
|
||||
test_id = csvreader.get_column(df, 'test-id')[0]
|
||||
for column in columns:
|
||||
try:
|
||||
column_data = csvreader.get_column(df, column)[0]
|
||||
duts[column] = column_data
|
||||
except:
|
||||
print('no column named %s' % column)
|
||||
try:
|
||||
test_tag[test_id] = (csvreader.get_column(df, 'test-tag')[0])
|
||||
test_tag[test_id] = csvreader.get_column(df, 'test-tag')[0]
|
||||
except:
|
||||
notesttag = True
|
||||
print('no test-id')
|
||||
pass_fail = Counter(csvreader.get_column(df, 'pass/fail'))
|
||||
test_pass_fail.append(pass_fail)
|
||||
dut_hw = csvreader.get_column(df, 'dut-hw-version')[0]
|
||||
dut_sw = csvreader.get_column(df, 'dut-sw-version')[0]
|
||||
dut_model = csvreader.get_column(df, 'dut-model-num')[0]
|
||||
dut_serial = csvreader.get_column(df, 'dut-serial-num')[0]
|
||||
subtest_pass = csvreader.get_column(df, 'Subtest-Pass')
|
||||
subtest_fail = csvreader.get_column(df, 'Subtest-Fail')
|
||||
for result in subtest_pass:
|
||||
@@ -333,10 +312,6 @@ class GhostRequest:
|
||||
subtest_pass_fail_list['PASS'] = subtest_pass_total
|
||||
subtest_pass_fail_list['FAIL'] = subtest_fail_total
|
||||
subtest_pass_fail.append(subtest_pass_fail_list)
|
||||
if notesttag:
|
||||
duts = [dut_serial, dut_hw, dut_sw, dut_model, test_rig, test_tag]
|
||||
else:
|
||||
duts = [dut_serial, dut_hw, dut_sw, dut_model, test_rig]
|
||||
times_append = csvreader.get_column(df, 'Date')
|
||||
if len(times_append) == 0:
|
||||
print(LookupError("%s/kpi.csv has no time points" % target_folder))
|
||||
@@ -352,13 +327,12 @@ class GhostRequest:
|
||||
text = text + 'Tests passed: 0<br />' \
|
||||
'Tests failed : 0<br />' \
|
||||
'Percentage of tests passed: Not Applicable<br />'
|
||||
testbeds.append(test_rig)
|
||||
testbeds.append(duts['test-rig'])
|
||||
if testbed is None:
|
||||
testbed = test_rig
|
||||
testbed = duts['test-rig']
|
||||
|
||||
if test_run is None:
|
||||
test_run = now.strftime('%B-%d-%Y-%I-%M-%p-report')
|
||||
|
||||
local_path = '/home/%s/%s/%s/%s' % (user_push, customer, testbed, test_run)
|
||||
|
||||
transport = paramiko.Transport(ghost_host, port)
|
||||
@@ -391,9 +365,9 @@ class GhostRequest:
|
||||
url = 'http://%s/%s/%s/%s/%s/%s' % (
|
||||
ghost_host, customer.strip('/'), testbed, test_run, target_folder, 'index.html')
|
||||
webpages.append('<a href="%s">HTML</a>' % url)
|
||||
webpagesandpdfsappend = dict()
|
||||
webpagesandpdfsappend[test_id] = pdfs + webpages
|
||||
webpagesandpdfs.append(webpagesandpdfsappend)
|
||||
web_pages_and_pdfs_append = dict()
|
||||
web_pages_and_pdfs_append[test_id] = pdfs + webpages
|
||||
web_pages_and_pdfs.append(web_pages_and_pdfs_append)
|
||||
scp_push.close()
|
||||
self.upload_images(target_folder)
|
||||
for image in self.images:
|
||||
@@ -413,9 +387,9 @@ class GhostRequest:
|
||||
pass
|
||||
|
||||
low_priority = csvreader.filter_df(results, 'test-priority', 'less than', 94)
|
||||
print('Low Priority results %s' % len(low_priority))
|
||||
high_priority = csvreader.filter_df(results, 'test-priority', 'greater than or equal to', 95)
|
||||
high_priority_list.append(high_priority)
|
||||
|
||||
low_priority_list.append(low_priority)
|
||||
|
||||
except:
|
||||
@@ -423,11 +397,10 @@ class GhostRequest:
|
||||
target_folders.remove(target_folder)
|
||||
failuredict = dict()
|
||||
failuredict[target_folder] = ['Failure']
|
||||
webpagesandpdfs.append(failuredict)
|
||||
web_pages_and_pdfs.append(failuredict)
|
||||
if len(times) == 0:
|
||||
return ArithmeticError("There are no datapoints in any folders passed into Ghost")
|
||||
|
||||
|
||||
test_pass_fail_results = sum((Counter(test) for test in test_pass_fail), Counter())
|
||||
subtest_pass_fail_results = sum((Counter(test) for test in subtest_pass_fail), Counter())
|
||||
|
||||
@@ -443,14 +416,17 @@ class GhostRequest:
|
||||
high_priority = csvreader.concat(high_priority_list)
|
||||
low_priority = csvreader.concat(low_priority_list)
|
||||
|
||||
high_priority = csvreader.get_columns(high_priority,
|
||||
['Short Description', 'Score', 'Test Details'])
|
||||
if len(high_priority) > 0:
|
||||
high_priority = csvreader.get_columns(high_priority,
|
||||
['Short Description', 'Score', 'Test Details'])
|
||||
low_priority = csvreader.get_columns(low_priority,
|
||||
['Short Description', 'Score', 'Test Details'])
|
||||
high_priority.append(['Total Passed', test_pass_fail_results['PASS'], 'Total subtests passed during this run'])
|
||||
high_priority.append(['Total Failed', test_pass_fail_results['FAIL'], 'Total subtests failed during this run'])
|
||||
high_priority.append(['Subtests Passed', subtest_pass_fail_results['PASS'], 'Total subtests passed during this run'])
|
||||
high_priority.append(['Subtests Failed', subtest_pass_fail_results['FAIL'], 'Total subtests failed during this run'])
|
||||
high_priority.append(
|
||||
['Subtests Passed', subtest_pass_fail_results['PASS'], 'Total subtests passed during this run'])
|
||||
high_priority.append(
|
||||
['Subtests Failed', subtest_pass_fail_results['FAIL'], 'Total subtests failed during this run'])
|
||||
|
||||
if title is None:
|
||||
title = end_time.strftime('%B %d, %Y %I:%M %p report')
|
||||
@@ -461,15 +437,6 @@ class GhostRequest:
|
||||
target_files.append(folder.split('/')[-1] + '/kpi.csv')
|
||||
if self.debug:
|
||||
print('Target files: %s' % target_files)
|
||||
grafana.create_custom_dashboard(target_csvs=target_files,
|
||||
title=title,
|
||||
datasource=grafana_datasource,
|
||||
bucket=grafana_bucket,
|
||||
from_date=start_time,
|
||||
to_date=end_time.strftime('%Y-%m-%d %H:%M:%S'),
|
||||
pass_fail='GhostRequest',
|
||||
testbed=testbeds[0],
|
||||
test_tag=test_tag)
|
||||
|
||||
if self.influx_token is not None:
|
||||
influxdb = RecordInflux(_influx_host=self.influx_host,
|
||||
@@ -520,22 +487,28 @@ class GhostRequest:
|
||||
for tag in list(set(test_tag.values())):
|
||||
print(tag)
|
||||
test_tag_table += (
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Test Tag</td>' \
|
||||
'<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' % tag)
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Test Tag</td><td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' % tag)
|
||||
dut_table_column_names = {'test-rig': 'Testbed',
|
||||
'dut-hw-version': 'DUT HW',
|
||||
'dut-sw-version': 'DUT SW',
|
||||
'dut-model-num': 'DUT Model',
|
||||
'dut-serial-num': 'DUT Serial'}
|
||||
dut_table_columns = ''
|
||||
for column in columns:
|
||||
if column in dut_table_column_names.keys():
|
||||
column_name = dut_table_column_names[column]
|
||||
else:
|
||||
column_name = column
|
||||
dut_table_columns += (
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td><td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' % (
|
||||
column_name, duts[column])
|
||||
)
|
||||
|
||||
dut_table = '<table width="700px" border="1" cellpadding="2" cellspacing="0" ' \
|
||||
'style="border-color: gray; border-style: solid; border-width: 1px; "><tbody>' \
|
||||
'<tr><th colspan="2">Test Information</th></tr>' \
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Testbed</td>' \
|
||||
'<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
|
||||
'%s' \
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">DUT_HW</td>' \
|
||||
'<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">DUT_SW</td>' \
|
||||
'<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">DUT model</td>' \
|
||||
'<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">DUT Serial</td>' \
|
||||
'<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
|
||||
'%s' \
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Tests passed</td>' \
|
||||
'<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Tests failed</td>' \
|
||||
@@ -544,13 +517,13 @@ class GhostRequest:
|
||||
'<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
|
||||
'<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Subtests failed</td>' \
|
||||
'<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' % (
|
||||
duts[4], test_tag_table, duts[1], duts[2], duts[3], duts[0], test_pass_fail_results['PASS'],
|
||||
dut_table_columns, test_tag_table, test_pass_fail_results['PASS'],
|
||||
test_pass_fail_results['FAIL'], subtest_pass_total, subtest_fail_total)
|
||||
|
||||
dut_table = dut_table + '</tbody></table>'
|
||||
text = text + dut_table
|
||||
|
||||
for dictionary in webpagesandpdfs:
|
||||
for dictionary in web_pages_and_pdfs:
|
||||
text += list(dictionary.keys())[0] + ' report: '
|
||||
for value in dictionary.values():
|
||||
for webpage in value:
|
||||
@@ -565,6 +538,20 @@ class GhostRequest:
|
||||
text = text + 'High priority results: %s' % csvreader.to_html(high_priority)
|
||||
|
||||
if grafana_token is not None:
|
||||
grafana = GrafanaRequest(grafana_token,
|
||||
grafana_host,
|
||||
grafanajson_port=grafana_port
|
||||
)
|
||||
print(test_tag)
|
||||
grafana.create_custom_dashboard(target_csvs=target_files,
|
||||
title=title,
|
||||
datasource=grafana_datasource,
|
||||
bucket=grafana_bucket,
|
||||
from_date=start_time,
|
||||
to_date=end_time.strftime('%Y-%m-%d %H:%M:%S'),
|
||||
pass_fail='GhostRequest',
|
||||
testbed=testbeds[0],
|
||||
test_tag=test_tag)
|
||||
# get the details of the dashboard through the API, and set the end date to the youngest KPI
|
||||
grafana.list_dashboards()
|
||||
|
||||
@@ -577,6 +564,4 @@ class GhostRequest:
|
||||
text = text + 'Low priority results: %s' % csvreader.to_html(low_priority)
|
||||
|
||||
self.create_post(title=title,
|
||||
text=text,
|
||||
tags='custom',
|
||||
authors=authors)
|
||||
text=text)
|
||||
|
||||
@@ -170,8 +170,8 @@ class GrafanaRequest:
|
||||
if graph_group is not None:
|
||||
graphgroup = ('|> filter(fn: (r) => r["Graph-Group"] == "%s")\n' % graph_group)
|
||||
query += graphgroup
|
||||
if test_tag is not None:
|
||||
graphgroup = ('|> filter(fn: (r) => r["Test-Tag"] == "%s")\n' % test_tag)
|
||||
if test_tag is not None and len(test_tag) > 0:
|
||||
graphgroup = ('|> filter(fn: (r) => r["Test Tag"] == "%s")\n' % test_tag)
|
||||
query += graphgroup
|
||||
if testbed is not None:
|
||||
query += ('|> filter(fn: (r) => r["testbed"] == "%s")\n' % testbed)
|
||||
|
||||
Reference in New Issue
Block a user