From e28bbb92a967e565daca23dd54be8793bcabc3f0 Mon Sep 17 00:00:00 2001 From: Chuck SmileyRekiere Date: Sat, 17 Oct 2020 17:40:24 -0600 Subject: [PATCH] test_l3_longevity.py csv_processor.py: summary and raw csv files created --- py-scripts/csv_processor.py | 60 ++++++++++++++++++++------------- py-scripts/test_l3_longevity.py | 12 +++++-- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/py-scripts/csv_processor.py b/py-scripts/csv_processor.py index 15e4bf9d..734bf15e 100755 --- a/py-scripts/csv_processor.py +++ b/py-scripts/csv_processor.py @@ -4,16 +4,16 @@ import sys import os import argparse -import time -import datetime -import subprocess -import re -import csv -import time -import operator +#import time +#import datetime +#import subprocess +#import re +#import csv +#import time +#import operator import pandas as pd -import matplotlib.pyplot as plt -import numpy as np +#import matplotlib.pyplot as plt +#import numpy as np #https://pandas.pydata.org/pandas-docs/stable/user_guide/visualization.html #https://queirozf.com/entries/pandas-dataframe-plot-examples-with-matplotlib-pyplot @@ -30,7 +30,7 @@ class L3CSVParcer(): def __init__(self,csv_file): - # left this in + # left this in for testing '''csv_obj = open(csv_file, 'r') csv_reader = csv.reader(csv_obj) print(csv_reader) @@ -39,21 +39,38 @@ class L3CSVParcer(): if row[1] == 'rx': print(row)''' - include = ['Time','Monitor','least','most','average','drop'] + include_summary = ['Time epoch','Time','Monitor','least','most','average'] self.csv_file = csv_file - df = pd.read_csv(self.csv_file,header = 0, usecols = lambda column : any(substr in column for substr in include)) + df_s = pd.read_csv(self.csv_file,header = 0, usecols = lambda column : any(substr in column for substr in include_summary)) + + print('{}'.format(csv_file)) + csv_file_summary = self.csv_file.replace('results_','results_summary_') + + df_s.to_csv(csv_file_summary, index = False, header=True) + + include_raw = ['Time epoch','Time','Monitor','LT','MT'] + self.csv_file = csv_file + df_r = pd.read_csv(self.csv_file,header = 0, usecols = lambda column : any(substr in column for substr in include_raw)) + + csv_file_raw = self.csv_file.replace('results_','results_raw_') + df_r.to_csv(csv_file_raw, index = False, header=True) + + '''df_rx_delta = df_r.loc[df['Monitor'] == 'rx_delta'] + + df_rx_delta.plot(x='Time epoch', y='average_rx_data') + plt.show() total_cols = len(df.axes[0]) - #print(total_cols) - #self.df = pd.read_csv('longevity_results_08_12_2020_10_19.csv') - #print("df:{}".format(df)) print(df.columns) print(df.loc[df['Monitor'] == 'rx_delta']) print(df.loc[df['Monitor'] == 'rx']) + print(df.loc[df['Monitor'] == 'rx_delta', df.columns != 'Time']) + + df_rx_delta = df.loc[df['Monitor'] == 'rx_delta'] print(df_rx_delta.describe()) @@ -111,17 +128,12 @@ class L3CSVParcer(): #print(df_LT_rx_delta_mean) x = np.linspace(0, 20, 100) plt.plot(x, np.sin(x)) - plt.show() - - - - - + plt.show()''' def main(): - debug_on = False + #debug_on = False parser = argparse.ArgumentParser( prog='quick_test.py', formatter_class=argparse.RawTextHelpFormatter, @@ -140,12 +152,12 @@ def main(): args = parser.parse_args() - debug_on = args.debug + #debug_on = args.debug if args.infile: csv_file_name = args.infile - l3_csv_analysis = L3CSVParcer(csv_file_name) + L3CSVParcer(csv_file_name) diff --git a/py-scripts/test_l3_longevity.py b/py-scripts/test_l3_longevity.py index 9aa448e4..69f0153f 100755 --- a/py-scripts/test_l3_longevity.py +++ b/py-scripts/test_l3_longevity.py @@ -127,7 +127,7 @@ class L3VariableTime(LFCliBase): return endp_rx_map, endp_rx_drop_map def time_stamp(self): - return time.strftime('%Y-%m-%d %H %M %S', time.localtime(self.epoch_time)) + return time.strftime('%m_%d_%Y_%H_%M_%S', time.localtime(self.epoch_time)) def __record_rx_dropped_percent(self,rx_drop_percent): @@ -992,9 +992,9 @@ class L3VariableTime(LFCliBase): def csv_generate_column_headers(self): csv_rx_headers = ['Time epoch','Time','Monitor'] for i in range(1,6): - csv_rx_headers.append("least_rx_data_bytes {}".format(i)) + csv_rx_headers.append("least_rx_data_bytes_{}".format(i)) for i in range(1,6): - csv_rx_headers.append("most_rx_data_bytes{}".format(i)) + csv_rx_headers.append("most_rx_data_bytes_{}".format(i)) csv_rx_headers.append("average_rx_data_bytes") return csv_rx_headers @@ -1313,6 +1313,12 @@ python3 test_l3_longevity.py --cisco_ctlr 192.168.100.112 --cisco_dfs True --mgr print("stop test failed") print(ip_var_test.get_fail_message()) + try: + sub_output = subprocess.run(["./csv_processor.py", "--infile",csv_outfile],capture_output=True, check=True) + pss = sub_output.stdout.decode('utf-8', 'ignore') + print(pss) + except Exception as e: + print("Exception: {} failed creating summary and raw for {}, are all packages installed , pandas?".format(e,csv_outfile)) print("Pausing 30 seconds after run for manual inspection before we clean up.") time.sleep(30)