mirror of
https://github.com/Telecominfraproject/wlan-lanforge-scripts.git
synced 2025-11-02 19:58:03 +00:00
kpi_csv_sq.py : make class based
csv_sqlite.py : naming changes Signed-off-by: Chuck SmileyRekiere <chuck.smileyrekiere@candelatech.com>
This commit is contained in:
Chuck SmileyRekiere
@@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
'''
|
||||
File: will search sub diretories for kpi.csv and place the data into an sqlite database
|
||||
|
||||
File: will search path recursivly for kpi.csv and place into sqlite database
|
||||
Usage: csv_sqlite.py --path <path to directories to traverse> --database <name of database>
|
||||
'''
|
||||
|
||||
@@ -19,39 +17,28 @@ class csv_to_sqlite():
|
||||
def __init__(self,
|
||||
_path = '.',
|
||||
_file = 'kpi.csv',
|
||||
_database = 'test_db'):
|
||||
_database = 'qa_db',
|
||||
_table = 'qa_table'):
|
||||
self.path = _path
|
||||
self.file = _file
|
||||
self.database = _database
|
||||
self.table = _table
|
||||
|
||||
# information on sqlite database
|
||||
# https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html
|
||||
def store(self):
|
||||
print(self.path)
|
||||
path = Path(self.path)
|
||||
print(path)
|
||||
#path = Path('./test_data3')
|
||||
#path = Path('test_data3')
|
||||
#print(path)
|
||||
#quit(1)
|
||||
|
||||
kpi_list = list(path.glob('**/{}'.format(self.file)))
|
||||
#kpi_list = list(path.glob('**/kpi.csv'))
|
||||
print(kpi_list)
|
||||
print("lenth kpi_list {}".format(len(kpi_list)))
|
||||
|
||||
df = pd.DataFrame()
|
||||
|
||||
for kpi in kpi_list:
|
||||
# load data
|
||||
append_df = pd.read_csv(kpi, sep='\t')
|
||||
df = df.append(append_df, ignore_index=True)
|
||||
|
||||
# information on sqlite database
|
||||
# https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html
|
||||
|
||||
print(self.database)
|
||||
conn = sqlite3.connect(self.database)
|
||||
#data may be appended setting if_exists='append'
|
||||
df.to_sql("dp_table",conn,if_exists='replace')
|
||||
df.to_sql(self.table,conn,if_exists='replace')
|
||||
conn.close()
|
||||
|
||||
def main():
|
||||
@@ -63,7 +50,6 @@ def main():
|
||||
read kpi.csv into sqlit database
|
||||
|
||||
''',
|
||||
|
||||
description='''\
|
||||
File: will search path recursivly for kpi.csv and place into sqlite database
|
||||
Usage: csv_sqlite.py --path <path to directories to traverse> --database <name of database>
|
||||
@@ -71,18 +57,21 @@ Usage: csv_sqlite.py --path <path to directories to traverse> --database <name o
|
||||
''')
|
||||
parser.add_argument('--path', help='--path ./top directory path to kpi',required=True)
|
||||
parser.add_argument('--file', help='--file kpi.csv',default='kpi.csv')
|
||||
parser.add_argument('--database', help='--database qa_test_db',default='qa_test_db')
|
||||
parser.add_argument('--database', help='--database qa_db',default='qa_db')
|
||||
parser.add_argument('--table', help='--table qa_table',default='qa_table')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
__path = args.path
|
||||
__file = args.file
|
||||
__database = args.database
|
||||
__table = args.table
|
||||
|
||||
csv_sqlite = csv_to_sqlite(
|
||||
_path = __path,
|
||||
_file = __file,
|
||||
_database = __database)
|
||||
_database = __database,
|
||||
_table= __table)
|
||||
|
||||
csv_sqlite.store()
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Run this app with `python app.py` and
|
||||
'''
|
||||
File: use kpi.csv placed in sql database, create png of historical kpi and present on dashboard
|
||||
Usage: csv_sqlite.py --path <path to directories to traverse> --database <name of database>
|
||||
'''
|
||||
# visit http://127.0.0.1:8050/ in your web browser.
|
||||
|
||||
import os
|
||||
@@ -10,166 +12,166 @@ import dash_html_components as html
|
||||
import plotly.express as px
|
||||
import pandas as pd
|
||||
import sqlite3
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# Any style components can be used
|
||||
external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
|
||||
|
||||
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
|
||||
class csv_sqlite_dash():
|
||||
def __init__(self,
|
||||
_path = '.',
|
||||
_file = 'kpi.csv',
|
||||
_database = 'qa_db',
|
||||
_table = 'qa_table'):
|
||||
self.path = _path
|
||||
self.file = _file
|
||||
self.database = _database
|
||||
self.table = _table
|
||||
self.kpi_list = []
|
||||
self.html_list = []
|
||||
self.conn = None
|
||||
self.df = pd.DataFrame()
|
||||
self.plot_figure = []
|
||||
children_div = []
|
||||
|
||||
# get a list of the kpi.csv files not the path needs to be bassed in
|
||||
#path = Path('./2021-07-31-03-00-01_lf_check')
|
||||
#path = Path('./test_data3')
|
||||
path = Path('./lf_check_2')
|
||||
# information on sqlite database
|
||||
# https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html
|
||||
def store(self):
|
||||
print("reading kpi and storing in db {}".format(self.database))
|
||||
path = Path(self.path)
|
||||
#self.kpi_list = list(path.glob('**/{}'.format(self.file)))
|
||||
self.kpi_list = list(path.glob('**/kpi.csv'))
|
||||
|
||||
kpi_list = list(path.glob('**/kpi.csv'))
|
||||
print("kpi_list {}".format(kpi_list))
|
||||
self.html_list = list(path.glob('**/index.html')) # the html is only index.html
|
||||
print("html_list: {}".format(html_list))
|
||||
|
||||
html_list = list(path.glob('**/index.html')) # the html is only index.html
|
||||
print("html_list: {}".format(html_list))
|
||||
for kpi in self.kpi_list: #TODO note empty kpi.csv failed test
|
||||
df_kpi_tmp = pd.read_csv(kpi, sep='\t')
|
||||
df_kpi_tmp['kpi_path'] = str(kpi).replace('kpi.csv','') # only store the path to the kpi.csv file
|
||||
df_kpi_tmp = df_kpi_tmp.append(df_kpi_tmp, ignore_index=True)
|
||||
self.df = df.append(df_kpi_tmp, ignore_index=True)
|
||||
|
||||
df = pd.DataFrame()
|
||||
self.conn = sqlite3.connect(self.database)
|
||||
#data may be appended setting if_exists='append'
|
||||
self.df.to_sql(self.table,self.conn,if_exists='replace')
|
||||
self.conn.close()
|
||||
|
||||
# duplicates the store since the the png are put back into the directory where the kpi are gathered
|
||||
def generate_png(self):
|
||||
print("generating png files")
|
||||
if not self.kpi_list:
|
||||
self.store()
|
||||
if not self.kpi_list:
|
||||
print("no kpi.csv found, check input paths, exiting")
|
||||
exit(1)
|
||||
|
||||
# if there is no KPI then there is not an indication that
|
||||
# a test was run so need to fix
|
||||
#not if there is a kpi then there is an index.html
|
||||
for kpi in kpi_list:
|
||||
# load data
|
||||
print("kpi {}".format(kpi))
|
||||
df_kpi_tmp = pd.read_csv(kpi, sep='\t') # remove the index
|
||||
df_kpi_tmp['kpi_path'] = str(kpi).replace('kpi.csv','') # only store the path to the kpi.csv file
|
||||
df_kpi_tmp = df_kpi_tmp.append(df_kpi_tmp, ignore_index=True)
|
||||
df = df.append(df_kpi_tmp, ignore_index=True)
|
||||
#https://datacarpentry.org/python-ecology-lesson/09-working-with-sql/index.html
|
||||
self.conn = sqlite3.connect(self.database)
|
||||
# df3 is just a name
|
||||
df3 = pd.read_sql_query("SELECT * from dp_table" ,self.conn)
|
||||
#print(df3.head())
|
||||
self.conn.close()
|
||||
|
||||
#df.reset_index(drop=True)
|
||||
print("df {data}".format(data=df))
|
||||
# graph group and test-tag are used for detemining the graphs
|
||||
graph_group_list = list(df3['Graph-Group'])
|
||||
graph_group_list = list(set(graph_group_list)) #remove duplicates
|
||||
|
||||
test_tag_list = list(df3['test-tag'])
|
||||
test_tag_list = list(set(test_tag_list))
|
||||
|
||||
# information on sqlite database
|
||||
# https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html
|
||||
for test_tag in test_tag_list:
|
||||
for group in graph_group_list:
|
||||
df_tmp = df3.loc[(df3['Graph-Group'] == str(group)) & (df3['test-tag'] == str(test_tag))]
|
||||
if df_tmp.empty == False:
|
||||
kpi_fig = (px.scatter(df_tmp, x="Date", y="numeric-score",
|
||||
color="short-description", hover_name="short-description",
|
||||
size_max=60)).update_traces(mode='lines+markers')
|
||||
|
||||
# write to database
|
||||
conn = sqlite3.connect("qa_db")
|
||||
#df.to_sql("dp_table",conn,if_exists='append')
|
||||
df.to_sql("dp_table",conn,if_exists='replace')
|
||||
conn.close()
|
||||
# remove duplicates from
|
||||
test_rig_list = list(df_tmp['test-rig'])
|
||||
test_rig = list(set(test_rig_list))
|
||||
|
||||
test_id_list = list(df_tmp['test-id'])
|
||||
test_id = list(set(test_id_list))
|
||||
|
||||
#https://datacarpentry.org/python-ecology-lesson/09-working-with-sql/index.html
|
||||
conn = sqlite3.connect("qa_db")
|
||||
df3 = pd.read_sql_query("SELECT * from dp_table" ,conn)
|
||||
#print(df3.head())
|
||||
conn.close()
|
||||
kpi_path_list = list(df_tmp['kpi_path'])
|
||||
kpi_path = list(set(kpi_path_list))
|
||||
|
||||
graph_group_list = list(df3['Graph-Group'])
|
||||
graph_group_list = list(set(graph_group_list)) #remove duplicates
|
||||
units_list = list(df_tmp['Units'])
|
||||
units = list(set(units_list))
|
||||
|
||||
test_tag_list = list(df3['test-tag'])
|
||||
test_tag_list = list(set(test_tag_list))
|
||||
kpi_fig.update_layout(
|
||||
title="{} : {} : {} : {}".format(test_id[0], group, test_tag, test_rig[0]),
|
||||
xaxis_title="Time",
|
||||
yaxis_title="{}".format(units[0]),
|
||||
xaxis = {'type' : 'date'}
|
||||
)
|
||||
# save the figure - this may need to be re-written
|
||||
print("kpi_path:{}".format(df_tmp['kpi_path']))
|
||||
png_path = os.path.join(kpi_path[0],"{}_{}_{}_{}_kpi.png".format(test_id[0], group, test_tag, test_rig[0]))
|
||||
print("png_path {}".format(png_path))
|
||||
kpi_fig.write_image(png_path,scale=1,width=1200,height=350)
|
||||
# store the image so it may be displayed
|
||||
self.children_div.append(dcc.Graph(figure=kpi_fig))
|
||||
|
||||
i = 0
|
||||
plot_figure = []
|
||||
children_div = []
|
||||
for test_tag in test_tag_list:
|
||||
for group in graph_group_list:
|
||||
df_tmp = df3.loc[(df3['Graph-Group'] == str(group)) & (df3['test-tag'] == str(test_tag))]
|
||||
#print("test_tag: {} group: {} ".format(test_tag,group))
|
||||
#print("df_tmp[{}]: {}".format(i,df_tmp))
|
||||
i = i + 1
|
||||
# access from server
|
||||
# https://stackoverflow.com/questions/61678129/how-to-access-a-plotly-dash-app-server-via-lan
|
||||
def show(self):
|
||||
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
|
||||
|
||||
if df_tmp.empty == False:
|
||||
append_fig = (px.scatter(df_tmp, x="Date", y="numeric-score",
|
||||
color="short-description", hover_name="short-description",
|
||||
size_max=60)).update_traces(mode='lines+markers')
|
||||
print("{}".format(df_tmp['Graph-Group']))
|
||||
app.layout = html.Div([
|
||||
html.H1(children= "LANforge Testing",className="lanforge",
|
||||
style={'color':'green','text-align':'center'}),
|
||||
html.H2(children= "Test Set #1",className="ts1",
|
||||
style={'color':'#00361c','text-align':'left'}),
|
||||
# images_div is already a list, children = a list of html components
|
||||
html.Div(children= self.children_div, style={"maxHeight": "480px", "overflow": "scroll"} ),
|
||||
html.H2(children= "Test Set #2",className="ts2",
|
||||
style={'color':'#00361c','text-align':'left'}),
|
||||
])
|
||||
|
||||
#print('{}'.format(type(append_fig)))
|
||||
# remove duplicates
|
||||
test_rig_list = list(df_tmp['test-rig'])
|
||||
test_rig = list(set(test_rig_list))
|
||||
def main():
|
||||
|
||||
test_id_list = list(df_tmp['test-id'])
|
||||
test_id = list(set(test_id_list))
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='kpi_csv_sq.py',
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
epilog='''\
|
||||
read kpi.csv into sqlite database , save png of history and preset on dashboard
|
||||
|
||||
kpi_path_list = list(df_tmp['kpi_path'])
|
||||
kpi_path = list(set(kpi_path_list))
|
||||
''',
|
||||
description='''\
|
||||
File: will search path recursivly for kpi.csv and place into sqlite database
|
||||
Usage: kpi_csv_sq.py --path <path to directories to traverse> --database <name of database>
|
||||
|
||||
# get graph labels from dataframe
|
||||
units_list = list(df_tmp['Units'])
|
||||
units = list(set(units_list))
|
||||
''')
|
||||
parser.add_argument('--path', help='--path ./top directory path to kpi',required=True)
|
||||
parser.add_argument('--file', help='--file kpi.csv',default='kpi.csv') #TODO is this needed
|
||||
parser.add_argument('--database', help='--database qa_test_db',default='qa_test_db')
|
||||
parser.add_argument('--store', help='--store , store kpi to db',action='store_true')
|
||||
parser.add_argument('--png', help='--png, may store kpi to db and generate png',action='store_true')
|
||||
parser.add_argument('--show', help='--show',action='store_true')
|
||||
|
||||
append_fig.update_layout(
|
||||
title="{} : {} : {} : {}".format(test_id[0], group, test_tag, test_rig[0]),
|
||||
xaxis_title="Time",
|
||||
yaxis_title="{}".format(units[0]),
|
||||
xaxis = {'type' : 'date'}
|
||||
)
|
||||
# save the figure - this may need to be re-written
|
||||
print("kpi_path:{}".format(df_tmp['kpi_path']))
|
||||
#exit(1)
|
||||
png_path = os.path.join(kpi_path[0],"{}_{}_{}_{}_kpi.png".format(test_id[0], group, test_tag, test_rig[0]))
|
||||
print("png_path {}".format(png_path))
|
||||
append_fig.write_image(png_path,scale=1,width=1200,height=350)
|
||||
children_div.append(dcc.Graph(figure=append_fig))
|
||||
# for now it was designed if there is a kpi.csv then there is an index.html
|
||||
#https://dash.plotly.com/urls
|
||||
#html_src = os.path.join(kpi_path[0],"index.html")
|
||||
# need to have a share available
|
||||
#local_share = 'file://LAPTOP-T8B2MBJD/Users/chuck/git/lanforge-scripts/py-scripts/sandbox/lf_check_2'
|
||||
#local_share = 'file://LAPTOP-T8B2MBJD/Users/chuck/git/lanforge-scripts/py-scripts/sandbox/'
|
||||
local_share = 'http:/127.0.0.1:8080/Users/chuck/git/lanforge-scripts/py-scripts/sandbox/'
|
||||
args = parser.parse_args()
|
||||
|
||||
html_src = os.path.join(local_share,kpi_path[0],"index.html")
|
||||
children_div.append(html.A('link{}'.format(i), href='http://192.168.95.6/html-reports/lf_check--2021-08-03-12-47-01.html', target='_blank'))
|
||||
children_div.append(html.Br())
|
||||
children_div.append(html.A(html_src, href=html_src, target='_blank'))
|
||||
children_div.append(html.Br())
|
||||
children_div.append(html.A('Navigate to google.com {}'.format(i), href='http://google.com', target='_blank'))
|
||||
#children_div.append(dcc.Location(id='dog',refresh=False))
|
||||
#children_div.append(dcc.Link('Link', href=html_src, target="_blank" ))
|
||||
# can add other information
|
||||
__path = args.path
|
||||
__file = args.file
|
||||
__database = args.database
|
||||
|
||||
csv_sqlite_dash = csv_sqlite_dash(
|
||||
_path = __path,
|
||||
_file = __file,
|
||||
_database = __database)
|
||||
if args.store:
|
||||
csv_sqlite_dash.store()
|
||||
if args.png:
|
||||
csv_sqlite_dash.generate_png()
|
||||
if args.show:
|
||||
csv_sqlite_dash.show()
|
||||
|
||||
# access from server
|
||||
# https://stackoverflow.com/questions/61678129/how-to-access-a-plotly-dash-app-server-via-lan
|
||||
|
||||
|
||||
app.layout = html.Div([
|
||||
# first instance
|
||||
html.H1(children= "LANforge Testing",className="lanforge",
|
||||
#style={'color':'#00361c','text-align':'center'}),
|
||||
style={'color':'green','text-align':'center'}),
|
||||
html.H2(children= "Test Set #1",className="ts1",
|
||||
style={'color':'#00361c','text-align':'left'}),
|
||||
#test html.Div(children= images_div ), # images_div is already a list, the children = a list
|
||||
html.Div(children= children_div ), # images_div is already a list, the children = a list
|
||||
html.H2(children= "Test Set #2",className="ts2",
|
||||
style={'color':'#00361c','text-align':'left'}),
|
||||
#test html.Div(children= images_div, style={"maxHeight": "480px","overflow": "scroll"})
|
||||
# danger::: this will cause components to show up twice
|
||||
#html.Div(children= children_div, style={"maxHeight": "480px","overflow": "scroll"})
|
||||
])
|
||||
|
||||
|
||||
# scroll bars
|
||||
'''
|
||||
app.layout = html.Div([
|
||||
# first instance
|
||||
html.H1(children= "LANforge Testing",className="lanforge",
|
||||
#style={'color':'#00361c','text-align':'center'}),
|
||||
style={'color':'green','text-align':'center'}),
|
||||
html.H2(children= "Test Set #1",className="ts1",
|
||||
style={'color':'#00361c','text-align':'left'}),
|
||||
html.Div(children= images_div, style={"maxHeight": "480px","overflow": "scroll"} ), # images_div is already a list , the children = a list
|
||||
html.H2(children= "Test Set #2",className="ts2",
|
||||
style={'color':'#00361c','text-align':'left'}),
|
||||
html.Div(children= images_div, style={"maxHeight": "480px","overflow": "scroll"})
|
||||
])
|
||||
|
||||
'''
|
||||
|
||||
if __name__ == '__main__':
|
||||
#app.run_server(debug=True)
|
||||
app.run_server(host= '0.0.0.0', debug=True) # host = '0.0.0.0' allows for remote access
|
||||
main()
|
||||
# host = '0.0.0.0' allows for remote access, local debug host = '127.0.0.1'
|
||||
# app.run_server(host= '0.0.0.0', debug=True)
|
||||
|
||||
Reference in New Issue
Block a user