kpi_csv_sq.py:

1. create display independently of generating the png
2. only generate the png files when --png is set
3. --show will using only the database to generate the display
	This was done to setup for a call back.  so generate the
	Display may be independent of generation of the png files
4. the generation of the png was kept with the generation of the display
as the display and the png need to match when generated.
Currently historical data is kept in the generated png

Signed-off-by: Chuck SmileyRekiere <chuck.smileyrekiere@candelatech.com>

py-scripts/sandbox/kpi_csv_sq.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
-File: use kpi.csv placed in sql database, create png of historical kpi and present on dashboard 
+File: read kpi.csv place in sql database, create png of historical kpi and present graph on dashboard
-Usage: csv_sqlite.py --path <path to directories to traverse> --database <name of database>
+Usage: csv_sqlite.py --path <path to directories to traverse> --database <name of database> --show --png  --store
 '''
 # visit http://127.0.0.1:8050/ in your web browser.
 
@@ -23,11 +23,13 @@ class csv_sqlite_dash():
                 _path = '.',
                 _file = 'kpi.csv',
                 _database = 'qa_db',
-                _table = 'qa_table'):
+                _table = 'qa_table',
+                _png = False):
         self.path = _path
         self.file = _file
         self.database = _database
         self.table = _table
+        self.png = _png
         self.kpi_list = []
         self.html_list = []
         self.conn = None
@@ -45,6 +47,9 @@ class csv_sqlite_dash():
         #self.kpi_list = list(path.glob('**/{}'.format(self.file)))
         self.kpi_list = list(path.glob('**/kpi.csv'))
 
+        if not self.kpi_list:
+            print("no new kpi.csv found, check input paths")
+
         for kpi in self.kpi_list: #TODO note empty kpi.csv failed test 
             df_kpi_tmp = pd.read_csv(kpi, sep='\t')  
             df_kpi_tmp['kpi_path'] = str(kpi).replace('kpi.csv','')  # only store the path to the kpi.csv file
@@ -57,11 +62,7 @@ class csv_sqlite_dash():
 
     # duplicates the store since the the png are put back into the directory where the kpi are gathered
     def generate_graph_png(self):
-        print("generating png files")
+        print("generating graphs to display")
-        if not self.kpi_list:
-            self.store()
-        if not self.kpi_list:
-            print("no new kpi.csv found, check input paths, will read database")
 
         #https://datacarpentry.org/python-ecology-lesson/09-working-with-sql/index.html-
         self.conn = sqlite3.connect(self.database)
@@ -103,18 +104,19 @@ class csv_sqlite_dash():
                             yaxis_title="{}".format(units_list[-1]),
                             xaxis = {'type' : 'date'}
                         )
-                        # save the figure - this may need to be re-written
+                        # save the figure - figures will be over written
-                        print("kpi_path:{}".format(df_tmp['kpi_path']))
+                        if self.png:
-                        png_path = os.path.join(kpi_path_list[-1],"{}_{}_{}_{}_kpi.png".format(test_id_list[-1], group, test_tag, test_rig))
+                            print("kpi_path:{}".format(df_tmp['kpi_path']))
-                        print("png_path {}".format(png_path))
+                            png_path = os.path.join(kpi_path_list[-1],"{}_{}_{}_{}_kpi.png".format(test_id_list[-1], group, test_tag, test_rig))
-                        kpi_fig.write_image(png_path,scale=1,width=1200,height=350)
+                            print("png_path {}".format(png_path))
+                            kpi_fig.write_image(png_path,scale=1,width=1200,height=350)
 
                         # use image from above to creat html display
                         self.children_div.append(dcc.Graph(figure=kpi_fig))                    
 
                         #TODO the link must be to a server to display html
-                        # WARNING: os.path.join will use the path for where the script is RUN which can be container.
+                        # WARNING: DO NOT USE os.path.join will use the path for where the script is RUN which can be container.
-                        # need to construct path to server manually. DO NOT USE os.path.join
+                        # need to construct path to server manually. 
                         #TODO need to work out the reporting paths - pass in path adjust
                         index_html_path = self.server + kpi_path_list[-1] + "index.html"
                         index_html_path = index_html_path.replace('/home/lanforge/','')
@@ -128,8 +130,7 @@ class csv_sqlite_dash():
     # access from server
     # https://stackoverflow.com/questions/61678129/how-to-access-a-plotly-dash-app-server-via-lan
     def show(self):
-        if not self.children_div:
+        self.generate_graph_png()
-            self.generate_graph_png()
         if not self.children_div:
             print("NOTE: test-tag may not be present in the kpi thus no results generated")
 
@@ -160,17 +161,17 @@ def main():
 
             ''',
         description='''\
-File: will search path recursivly for kpi.csv and place into sqlite database
+File: read kpi.csv place in sql database, create png of historical kpi and present graph on dashboard
-Usage: kpi_csv_sq.py --path <path to directories to traverse> --database <name of database>
+Usage: csv_sqlite.py --path <path to directories to traverse> --database <name of database> --show --png --store
 
         ''')
-    parser.add_argument('--path', help='--path ./top directory path to kpi',required=True)
+    parser.add_argument('--path', help='--path top directory path to kpi',default='')
     parser.add_argument('--file', help='--file kpi.csv',default='kpi.csv') #TODO is this needed
     parser.add_argument('--database', help='--database qa_test_db',default='qa_test_db')
     parser.add_argument('--table', help='--table qa_table',default='qa_table')
     parser.add_argument('--store', help='--store , store kpi to db',action='store_true')
-    parser.add_argument('--png', help='--png,  may store kpi to db and generate png',action='store_true')
+    parser.add_argument('--png', help='--png,  read db, generate png, generate display',action='store_true')
-    parser.add_argument('--show', help='--show',action='store_true')
+    parser.add_argument('--show', help='--show generate png and show dashboard',action='store_true')
     
     args = parser.parse_args()
 
@@ -178,15 +179,20 @@ Usage: kpi_csv_sq.py --path <path to directories to traverse> --database <name o
     __file = args.file
     __database = args.database
     __table = args.table
+    __png   = args.png
 
     print("config: path:{} file:{} database:{} table:{} store:{} png:{} show:{} "
         .format(__path,__file,__database,__table,args.store, args.png,args.show))
 
+    if(__path == '' and args.store == True):
+        print("--path <path of kpi.csv> must be entered if --store")
+
     csv_dash = csv_sqlite_dash(
                 _path = __path,
                 _file = __file,
                 _database = __database,
-                _table = __table)
+                _table = __table,
+                _png = __png)
     if args.store:
         csv_dash.store()
     if args.png:
@@ -195,7 +201,7 @@ Usage: kpi_csv_sq.py --path <path to directories to traverse> --database <name o
         csv_dash.show()
 
     if args.store == False and args.png == False and args.show == False:
-        print("Need to enter an action of --store  --png --show ")
+        print("Need to enter an action of --store --png --show ")
 
 if __name__ == '__main__':
     main()