test_l3_longevity.py csv_processor.py: summary and raw csv files created

py-scripts/csv_processor.py

@@ -4,16 +4,16 @@ import sys
 import os
 
 import argparse
-import time
-import datetime
-import subprocess
-import re
-import csv
-import time
-import operator
+#import time
+#import datetime
+#import subprocess
+#import re
+#import csv
+#import time
+#import operator
 import pandas as pd
-import matplotlib.pyplot as plt
-import numpy as np
+#import matplotlib.pyplot as plt
+#import numpy as np
 
 #https://pandas.pydata.org/pandas-docs/stable/user_guide/visualization.html
 #https://queirozf.com/entries/pandas-dataframe-plot-examples-with-matplotlib-pyplot
@@ -30,7 +30,7 @@ class L3CSVParcer():
     def __init__(self,csv_file):
 
 
-        # left this in
+        # left this in for testing
         '''csv_obj = open(csv_file, 'r')
         csv_reader = csv.reader(csv_obj)
         print(csv_reader)
@@ -39,21 +39,38 @@ class L3CSVParcer():
             if row[1] == 'rx':
                 print(row)'''
 
-        include = ['Time','Monitor','least','most','average','drop']
+        include_summary = ['Time epoch','Time','Monitor','least','most','average']
         self.csv_file = csv_file
-        df = pd.read_csv(self.csv_file,header = 0, usecols = lambda column : any(substr in column for substr in include))
+        df_s = pd.read_csv(self.csv_file,header = 0, usecols = lambda column : any(substr in column for substr in include_summary))
+
+        print('{}'.format(csv_file))
+        csv_file_summary = self.csv_file.replace('results_','results_summary_')
+
+        df_s.to_csv(csv_file_summary, index = False, header=True)
+
+        include_raw = ['Time epoch','Time','Monitor','LT','MT']
+        self.csv_file = csv_file
+        df_r = pd.read_csv(self.csv_file,header = 0, usecols = lambda column : any(substr in column for substr in include_raw))
+
+        csv_file_raw = self.csv_file.replace('results_','results_raw_')
+        df_r.to_csv(csv_file_raw, index = False, header=True)
+
+        '''df_rx_delta = df_r.loc[df['Monitor'] == 'rx_delta']
+
+        df_rx_delta.plot(x='Time epoch', y='average_rx_data')
+        plt.show()
 
         total_cols = len(df.axes[0])
 
-        #print(total_cols)
-        #self.df = pd.read_csv('longevity_results_08_12_2020_10_19.csv')
-        #print("df:{}".format(df))
 
         print(df.columns)
 
         print(df.loc[df['Monitor'] == 'rx_delta'])
         print(df.loc[df['Monitor'] == 'rx'])
 
+        print(df.loc[df['Monitor'] == 'rx_delta', df.columns != 'Time'])
+
+
         df_rx_delta = df.loc[df['Monitor'] == 'rx_delta']
 
         print(df_rx_delta.describe())
@@ -111,17 +128,12 @@ class L3CSVParcer():
         #print(df_LT_rx_delta_mean)
         x = np.linspace(0, 20, 100)
         plt.plot(x, np.sin(x))
-        plt.show()
-
-
-
-    
-
+        plt.show()'''
 
 
 def main():
 
-    debug_on = False
+    #debug_on = False
     parser = argparse.ArgumentParser(
         prog='quick_test.py',
         formatter_class=argparse.RawTextHelpFormatter,
@@ -140,12 +152,12 @@ def main():
 
     args = parser.parse_args()
 
-    debug_on = args.debug
+    #debug_on = args.debug
 
     if args.infile:
         csv_file_name = args.infile
 
-    l3_csv_analysis = L3CSVParcer(csv_file_name)
+    L3CSVParcer(csv_file_name)
 
 
 

py-scripts/test_l3_longevity.py

@@ -127,7 +127,7 @@ class L3VariableTime(LFCliBase):
         return endp_rx_map, endp_rx_drop_map
 
     def time_stamp(self):
-        return time.strftime('%Y-%m-%d %H %M %S', time.localtime(self.epoch_time))
+        return time.strftime('%m_%d_%Y_%H_%M_%S', time.localtime(self.epoch_time))
 
     def __record_rx_dropped_percent(self,rx_drop_percent):
 
@@ -992,9 +992,9 @@ class L3VariableTime(LFCliBase):
     def csv_generate_column_headers(self):
         csv_rx_headers = ['Time epoch','Time','Monitor']
         for i in range(1,6):
-            csv_rx_headers.append("least_rx_data_bytes {}".format(i))
+            csv_rx_headers.append("least_rx_data_bytes_{}".format(i))
         for i in range(1,6):
-            csv_rx_headers.append("most_rx_data_bytes{}".format(i))
+            csv_rx_headers.append("most_rx_data_bytes_{}".format(i))
         csv_rx_headers.append("average_rx_data_bytes")
         return csv_rx_headers
 
@@ -1313,6 +1313,12 @@ python3 test_l3_longevity.py --cisco_ctlr 192.168.100.112 --cisco_dfs True --mgr
         print("stop test failed")
         print(ip_var_test.get_fail_message())
          
+    try: 
+        sub_output = subprocess.run(["./csv_processor.py", "--infile",csv_outfile],capture_output=True, check=True)
+        pss = sub_output.stdout.decode('utf-8', 'ignore')
+        print(pss)
+    except Exception as e:
+        print("Exception: {} failed creating summary and raw for {}, are all packages installed , pandas?".format(e,csv_outfile))
 
     print("Pausing 30 seconds after run for manual inspection before we clean up.")
     time.sleep(30)