From aef76bc53cb095caca27215535fe8e7236e741ae Mon Sep 17 00:00:00 2001 From: Chuck SmileyRekiere Date: Thu, 5 Aug 2021 09:27:50 -0600 Subject: [PATCH] kpi_csv_sq.py: 1. create display independently of generating the png 2. only generate the png files when --png is set 3. --show will using only the database to generate the display This was done to setup for a call back. so generate the Display may be independent of generation of the png files 4. the generation of the png was kept with the generation of the display as the display and the png need to match when generated. Currently historical data is kept in the generated png Signed-off-by: Chuck SmileyRekiere --- py-scripts/sandbox/kpi_csv_sq.py | 54 ++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/py-scripts/sandbox/kpi_csv_sq.py b/py-scripts/sandbox/kpi_csv_sq.py index b471c276..d075b81e 100755 --- a/py-scripts/sandbox/kpi_csv_sq.py +++ b/py-scripts/sandbox/kpi_csv_sq.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' -File: use kpi.csv placed in sql database, create png of historical kpi and present on dashboard -Usage: csv_sqlite.py --path --database +File: read kpi.csv place in sql database, create png of historical kpi and present graph on dashboard +Usage: csv_sqlite.py --path --database --show --png --store ''' # visit http://127.0.0.1:8050/ in your web browser. @@ -23,11 +23,13 @@ class csv_sqlite_dash(): _path = '.', _file = 'kpi.csv', _database = 'qa_db', - _table = 'qa_table'): + _table = 'qa_table', + _png = False): self.path = _path self.file = _file self.database = _database self.table = _table + self.png = _png self.kpi_list = [] self.html_list = [] self.conn = None @@ -45,6 +47,9 @@ class csv_sqlite_dash(): #self.kpi_list = list(path.glob('**/{}'.format(self.file))) self.kpi_list = list(path.glob('**/kpi.csv')) + if not self.kpi_list: + print("no new kpi.csv found, check input paths") + for kpi in self.kpi_list: #TODO note empty kpi.csv failed test df_kpi_tmp = pd.read_csv(kpi, sep='\t') df_kpi_tmp['kpi_path'] = str(kpi).replace('kpi.csv','') # only store the path to the kpi.csv file @@ -57,11 +62,7 @@ class csv_sqlite_dash(): # duplicates the store since the the png are put back into the directory where the kpi are gathered def generate_graph_png(self): - print("generating png files") - if not self.kpi_list: - self.store() - if not self.kpi_list: - print("no new kpi.csv found, check input paths, will read database") + print("generating graphs to display") #https://datacarpentry.org/python-ecology-lesson/09-working-with-sql/index.html- self.conn = sqlite3.connect(self.database) @@ -103,18 +104,19 @@ class csv_sqlite_dash(): yaxis_title="{}".format(units_list[-1]), xaxis = {'type' : 'date'} ) - # save the figure - this may need to be re-written - print("kpi_path:{}".format(df_tmp['kpi_path'])) - png_path = os.path.join(kpi_path_list[-1],"{}_{}_{}_{}_kpi.png".format(test_id_list[-1], group, test_tag, test_rig)) - print("png_path {}".format(png_path)) - kpi_fig.write_image(png_path,scale=1,width=1200,height=350) + # save the figure - figures will be over written + if self.png: + print("kpi_path:{}".format(df_tmp['kpi_path'])) + png_path = os.path.join(kpi_path_list[-1],"{}_{}_{}_{}_kpi.png".format(test_id_list[-1], group, test_tag, test_rig)) + print("png_path {}".format(png_path)) + kpi_fig.write_image(png_path,scale=1,width=1200,height=350) # use image from above to creat html display self.children_div.append(dcc.Graph(figure=kpi_fig)) #TODO the link must be to a server to display html - # WARNING: os.path.join will use the path for where the script is RUN which can be container. - # need to construct path to server manually. DO NOT USE os.path.join + # WARNING: DO NOT USE os.path.join will use the path for where the script is RUN which can be container. + # need to construct path to server manually. #TODO need to work out the reporting paths - pass in path adjust index_html_path = self.server + kpi_path_list[-1] + "index.html" index_html_path = index_html_path.replace('/home/lanforge/','') @@ -128,8 +130,7 @@ class csv_sqlite_dash(): # access from server # https://stackoverflow.com/questions/61678129/how-to-access-a-plotly-dash-app-server-via-lan def show(self): - if not self.children_div: - self.generate_graph_png() + self.generate_graph_png() if not self.children_div: print("NOTE: test-tag may not be present in the kpi thus no results generated") @@ -160,17 +161,17 @@ def main(): ''', description='''\ -File: will search path recursivly for kpi.csv and place into sqlite database -Usage: kpi_csv_sq.py --path --database +File: read kpi.csv place in sql database, create png of historical kpi and present graph on dashboard +Usage: csv_sqlite.py --path --database --show --png --store ''') - parser.add_argument('--path', help='--path ./top directory path to kpi',required=True) + parser.add_argument('--path', help='--path top directory path to kpi',default='') parser.add_argument('--file', help='--file kpi.csv',default='kpi.csv') #TODO is this needed parser.add_argument('--database', help='--database qa_test_db',default='qa_test_db') parser.add_argument('--table', help='--table qa_table',default='qa_table') parser.add_argument('--store', help='--store , store kpi to db',action='store_true') - parser.add_argument('--png', help='--png, may store kpi to db and generate png',action='store_true') - parser.add_argument('--show', help='--show',action='store_true') + parser.add_argument('--png', help='--png, read db, generate png, generate display',action='store_true') + parser.add_argument('--show', help='--show generate png and show dashboard',action='store_true') args = parser.parse_args() @@ -178,15 +179,20 @@ Usage: kpi_csv_sq.py --path --database must be entered if --store") + csv_dash = csv_sqlite_dash( _path = __path, _file = __file, _database = __database, - _table = __table) + _table = __table, + _png = __png) if args.store: csv_dash.store() if args.png: @@ -195,7 +201,7 @@ Usage: kpi_csv_sq.py --path --database