diff --git a/py-scripts/sandbox/csv_sqlite.py b/py-scripts/sandbox/csv_sqlite.py index ec2b7e75..cb1f22b9 100644 --- a/py-scripts/sandbox/csv_sqlite.py +++ b/py-scripts/sandbox/csv_sqlite.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 ''' File: will search sub diretories for kpi.csv and place the data into an sqlite database - -File: will search path recursivly for kpi.csv and place into sqlite database Usage: csv_sqlite.py --path --database ''' @@ -19,39 +17,28 @@ class csv_to_sqlite(): def __init__(self, _path = '.', _file = 'kpi.csv', - _database = 'test_db'): + _database = 'qa_db', + _table = 'qa_table'): self.path = _path self.file = _file self.database = _database + self.table = _table + # information on sqlite database + # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html def store(self): - print(self.path) path = Path(self.path) - print(path) - #path = Path('./test_data3') - #path = Path('test_data3') - #print(path) - #quit(1) - kpi_list = list(path.glob('**/{}'.format(self.file))) - #kpi_list = list(path.glob('**/kpi.csv')) - print(kpi_list) - print("lenth kpi_list {}".format(len(kpi_list))) df = pd.DataFrame() - for kpi in kpi_list: - # load data append_df = pd.read_csv(kpi, sep='\t') df = df.append(append_df, ignore_index=True) - # information on sqlite database - # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html - print(self.database) conn = sqlite3.connect(self.database) #data may be appended setting if_exists='append' - df.to_sql("dp_table",conn,if_exists='replace') + df.to_sql(self.table,conn,if_exists='replace') conn.close() def main(): @@ -63,7 +50,6 @@ def main(): read kpi.csv into sqlit database ''', - description='''\ File: will search path recursivly for kpi.csv and place into sqlite database Usage: csv_sqlite.py --path --database @@ -71,18 +57,21 @@ Usage: csv_sqlite.py --path --database --database +''' # visit http://127.0.0.1:8050/ in your web browser. import os @@ -10,166 +12,166 @@ import dash_html_components as html import plotly.express as px import pandas as pd import sqlite3 +import argparse from pathlib import Path - +# Any style components can be used external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css'] -app = dash.Dash(__name__, external_stylesheets=external_stylesheets) +class csv_sqlite_dash(): + def __init__(self, + _path = '.', + _file = 'kpi.csv', + _database = 'qa_db', + _table = 'qa_table'): + self.path = _path + self.file = _file + self.database = _database + self.table = _table + self.kpi_list = [] + self.html_list = [] + self.conn = None + self.df = pd.DataFrame() + self.plot_figure = [] + children_div = [] -# get a list of the kpi.csv files not the path needs to be bassed in -#path = Path('./2021-07-31-03-00-01_lf_check') -#path = Path('./test_data3') -path = Path('./lf_check_2') + # information on sqlite database + # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html + def store(self): + print("reading kpi and storing in db {}".format(self.database)) + path = Path(self.path) + #self.kpi_list = list(path.glob('**/{}'.format(self.file))) + self.kpi_list = list(path.glob('**/kpi.csv')) -kpi_list = list(path.glob('**/kpi.csv')) -print("kpi_list {}".format(kpi_list)) + self.html_list = list(path.glob('**/index.html')) # the html is only index.html + print("html_list: {}".format(html_list)) -html_list = list(path.glob('**/index.html')) # the html is only index.html -print("html_list: {}".format(html_list)) + for kpi in self.kpi_list: #TODO note empty kpi.csv failed test + df_kpi_tmp = pd.read_csv(kpi, sep='\t') + df_kpi_tmp['kpi_path'] = str(kpi).replace('kpi.csv','') # only store the path to the kpi.csv file + df_kpi_tmp = df_kpi_tmp.append(df_kpi_tmp, ignore_index=True) + self.df = df.append(df_kpi_tmp, ignore_index=True) -df = pd.DataFrame() + self.conn = sqlite3.connect(self.database) + #data may be appended setting if_exists='append' + self.df.to_sql(self.table,self.conn,if_exists='replace') + self.conn.close() + # duplicates the store since the the png are put back into the directory where the kpi are gathered + def generate_png(self): + print("generating png files") + if not self.kpi_list: + self.store() + if not self.kpi_list: + print("no kpi.csv found, check input paths, exiting") + exit(1) -# if there is no KPI then there is not an indication that -# a test was run so need to fix -#not if there is a kpi then there is an index.html -for kpi in kpi_list: - # load data - print("kpi {}".format(kpi)) - df_kpi_tmp = pd.read_csv(kpi, sep='\t') # remove the index - df_kpi_tmp['kpi_path'] = str(kpi).replace('kpi.csv','') # only store the path to the kpi.csv file - df_kpi_tmp = df_kpi_tmp.append(df_kpi_tmp, ignore_index=True) - df = df.append(df_kpi_tmp, ignore_index=True) + #https://datacarpentry.org/python-ecology-lesson/09-working-with-sql/index.html + self.conn = sqlite3.connect(self.database) + # df3 is just a name + df3 = pd.read_sql_query("SELECT * from dp_table" ,self.conn) + #print(df3.head()) + self.conn.close() -#df.reset_index(drop=True) -print("df {data}".format(data=df)) + # graph group and test-tag are used for detemining the graphs + graph_group_list = list(df3['Graph-Group']) + graph_group_list = list(set(graph_group_list)) #remove duplicates + test_tag_list = list(df3['test-tag']) + test_tag_list = list(set(test_tag_list)) -# information on sqlite database -# https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html + for test_tag in test_tag_list: + for group in graph_group_list: + df_tmp = df3.loc[(df3['Graph-Group'] == str(group)) & (df3['test-tag'] == str(test_tag))] + if df_tmp.empty == False: + kpi_fig = (px.scatter(df_tmp, x="Date", y="numeric-score", + color="short-description", hover_name="short-description", + size_max=60)).update_traces(mode='lines+markers') -# write to database -conn = sqlite3.connect("qa_db") -#df.to_sql("dp_table",conn,if_exists='append') -df.to_sql("dp_table",conn,if_exists='replace') -conn.close() + # remove duplicates from + test_rig_list = list(df_tmp['test-rig']) + test_rig = list(set(test_rig_list)) + test_id_list = list(df_tmp['test-id']) + test_id = list(set(test_id_list)) -#https://datacarpentry.org/python-ecology-lesson/09-working-with-sql/index.html -conn = sqlite3.connect("qa_db") -df3 = pd.read_sql_query("SELECT * from dp_table" ,conn) -#print(df3.head()) -conn.close() + kpi_path_list = list(df_tmp['kpi_path']) + kpi_path = list(set(kpi_path_list)) -graph_group_list = list(df3['Graph-Group']) -graph_group_list = list(set(graph_group_list)) #remove duplicates + units_list = list(df_tmp['Units']) + units = list(set(units_list)) -test_tag_list = list(df3['test-tag']) -test_tag_list = list(set(test_tag_list)) + kpi_fig.update_layout( + title="{} : {} : {} : {}".format(test_id[0], group, test_tag, test_rig[0]), + xaxis_title="Time", + yaxis_title="{}".format(units[0]), + xaxis = {'type' : 'date'} + ) + # save the figure - this may need to be re-written + print("kpi_path:{}".format(df_tmp['kpi_path'])) + png_path = os.path.join(kpi_path[0],"{}_{}_{}_{}_kpi.png".format(test_id[0], group, test_tag, test_rig[0])) + print("png_path {}".format(png_path)) + kpi_fig.write_image(png_path,scale=1,width=1200,height=350) + # store the image so it may be displayed + self.children_div.append(dcc.Graph(figure=kpi_fig)) -i = 0 -plot_figure = [] -children_div = [] -for test_tag in test_tag_list: - for group in graph_group_list: - df_tmp = df3.loc[(df3['Graph-Group'] == str(group)) & (df3['test-tag'] == str(test_tag))] - #print("test_tag: {} group: {} ".format(test_tag,group)) - #print("df_tmp[{}]: {}".format(i,df_tmp)) - i = i + 1 + # access from server + # https://stackoverflow.com/questions/61678129/how-to-access-a-plotly-dash-app-server-via-lan + def show(self): + app = dash.Dash(__name__, external_stylesheets=external_stylesheets) - if df_tmp.empty == False: - append_fig = (px.scatter(df_tmp, x="Date", y="numeric-score", - color="short-description", hover_name="short-description", - size_max=60)).update_traces(mode='lines+markers') - print("{}".format(df_tmp['Graph-Group'])) + app.layout = html.Div([ + html.H1(children= "LANforge Testing",className="lanforge", + style={'color':'green','text-align':'center'}), + html.H2(children= "Test Set #1",className="ts1", + style={'color':'#00361c','text-align':'left'}), + # images_div is already a list, children = a list of html components + html.Div(children= self.children_div, style={"maxHeight": "480px", "overflow": "scroll"} ), + html.H2(children= "Test Set #2",className="ts2", + style={'color':'#00361c','text-align':'left'}), + ]) - #print('{}'.format(type(append_fig))) - # remove duplicates - test_rig_list = list(df_tmp['test-rig']) - test_rig = list(set(test_rig_list)) +def main(): - test_id_list = list(df_tmp['test-id']) - test_id = list(set(test_id_list)) + parser = argparse.ArgumentParser( + prog='kpi_csv_sq.py', + formatter_class=argparse.RawTextHelpFormatter, + epilog='''\ + read kpi.csv into sqlite database , save png of history and preset on dashboard - kpi_path_list = list(df_tmp['kpi_path']) - kpi_path = list(set(kpi_path_list)) + ''', + description='''\ +File: will search path recursivly for kpi.csv and place into sqlite database +Usage: kpi_csv_sq.py --path --database - # get graph labels from dataframe - units_list = list(df_tmp['Units']) - units = list(set(units_list)) + ''') + parser.add_argument('--path', help='--path ./top directory path to kpi',required=True) + parser.add_argument('--file', help='--file kpi.csv',default='kpi.csv') #TODO is this needed + parser.add_argument('--database', help='--database qa_test_db',default='qa_test_db') + parser.add_argument('--store', help='--store , store kpi to db',action='store_true') + parser.add_argument('--png', help='--png, may store kpi to db and generate png',action='store_true') + parser.add_argument('--show', help='--show',action='store_true') + + args = parser.parse_args() - append_fig.update_layout( - title="{} : {} : {} : {}".format(test_id[0], group, test_tag, test_rig[0]), - xaxis_title="Time", - yaxis_title="{}".format(units[0]), - xaxis = {'type' : 'date'} - ) - # save the figure - this may need to be re-written - print("kpi_path:{}".format(df_tmp['kpi_path'])) - #exit(1) - png_path = os.path.join(kpi_path[0],"{}_{}_{}_{}_kpi.png".format(test_id[0], group, test_tag, test_rig[0])) - print("png_path {}".format(png_path)) - append_fig.write_image(png_path,scale=1,width=1200,height=350) - children_div.append(dcc.Graph(figure=append_fig)) - # for now it was designed if there is a kpi.csv then there is an index.html - #https://dash.plotly.com/urls - #html_src = os.path.join(kpi_path[0],"index.html") - # need to have a share available - #local_share = 'file://LAPTOP-T8B2MBJD/Users/chuck/git/lanforge-scripts/py-scripts/sandbox/lf_check_2' - #local_share = 'file://LAPTOP-T8B2MBJD/Users/chuck/git/lanforge-scripts/py-scripts/sandbox/' - local_share = 'http:/127.0.0.1:8080/Users/chuck/git/lanforge-scripts/py-scripts/sandbox/' + __path = args.path + __file = args.file + __database = args.database - html_src = os.path.join(local_share,kpi_path[0],"index.html") - children_div.append(html.A('link{}'.format(i), href='http://192.168.95.6/html-reports/lf_check--2021-08-03-12-47-01.html', target='_blank')) - children_div.append(html.Br()) - children_div.append(html.A(html_src, href=html_src, target='_blank')) - children_div.append(html.Br()) - children_div.append(html.A('Navigate to google.com {}'.format(i), href='http://google.com', target='_blank')) - #children_div.append(dcc.Location(id='dog',refresh=False)) - #children_div.append(dcc.Link('Link', href=html_src, target="_blank" )) - # can add other information + csv_sqlite_dash = csv_sqlite_dash( + _path = __path, + _file = __file, + _database = __database) + if args.store: + csv_sqlite_dash.store() + if args.png: + csv_sqlite_dash.generate_png() + if args.show: + csv_sqlite_dash.show() -# access from server -# https://stackoverflow.com/questions/61678129/how-to-access-a-plotly-dash-app-server-via-lan - - -app.layout = html.Div([ - # first instance - html.H1(children= "LANforge Testing",className="lanforge", - #style={'color':'#00361c','text-align':'center'}), - style={'color':'green','text-align':'center'}), - html.H2(children= "Test Set #1",className="ts1", - style={'color':'#00361c','text-align':'left'}), - #test html.Div(children= images_div ), # images_div is already a list, the children = a list - html.Div(children= children_div ), # images_div is already a list, the children = a list - html.H2(children= "Test Set #2",className="ts2", - style={'color':'#00361c','text-align':'left'}), - #test html.Div(children= images_div, style={"maxHeight": "480px","overflow": "scroll"}) - # danger::: this will cause components to show up twice - #html.Div(children= children_div, style={"maxHeight": "480px","overflow": "scroll"}) -]) - - -# scroll bars -''' -app.layout = html.Div([ - # first instance - html.H1(children= "LANforge Testing",className="lanforge", - #style={'color':'#00361c','text-align':'center'}), - style={'color':'green','text-align':'center'}), - html.H2(children= "Test Set #1",className="ts1", - style={'color':'#00361c','text-align':'left'}), - html.Div(children= images_div, style={"maxHeight": "480px","overflow": "scroll"} ), # images_div is already a list , the children = a list - html.H2(children= "Test Set #2",className="ts2", - style={'color':'#00361c','text-align':'left'}), - html.Div(children= images_div, style={"maxHeight": "480px","overflow": "scroll"}) -]) - -''' - if __name__ == '__main__': - #app.run_server(debug=True) - app.run_server(host= '0.0.0.0', debug=True) # host = '0.0.0.0' allows for remote access + main() + # host = '0.0.0.0' allows for remote access, local debug host = '127.0.0.1' + # app.run_server(host= '0.0.0.0', debug=True) \ No newline at end of file