From b1ebef8f1955e1a3866a3ffa9133572a36cd75ec Mon Sep 17 00:00:00 2001 From: Dipti Date: Wed, 17 Feb 2021 12:47:40 -0800 Subject: [PATCH] added total_seconds function to calculate total seconds since epoch, test_time_elapsed column to help with dataframe comparison Signed-off-by: Dipti --- py-json/LANforge/lfcli_base.py | 16 +++++----- py-json/realm.py | 44 +++++++++++++-------------- py-scripts/test_ipv4_variable_time.py | 15 +++++---- 3 files changed, 36 insertions(+), 39 deletions(-) diff --git a/py-json/LANforge/lfcli_base.py b/py-json/LANforge/lfcli_base.py index 6233773a..f9fc84b5 100644 --- a/py-json/LANforge/lfcli_base.py +++ b/py-json/LANforge/lfcli_base.py @@ -580,6 +580,10 @@ class LFCliBase: def get_milliseconds(self, timestamp): return (timestamp - datetime.datetime(1970,1,1)).total_seconds()*1000 + def get_seconds(self, + timestamp): + return (timestamp - datetime.datetime(1970,1,1)).total_seconds() + def replace_special_char(self, str): @@ -626,15 +630,11 @@ class LFCliBase: #takes any format of a file and returns a dataframe of it def file_to_df(self,file_name): - return pd.read_csv(file_name) - #get difference of all common columns in 2 dataframes. store in + if file_name.split('.')[-1] == 'csv': + return pd.read_csv(file_name) + + def compare_two_df(self,dataframe_one=None,dataframe_two=None): - #get the columns to be compared - - - - - pass #return compared_df diff --git a/py-json/realm.py b/py-json/realm.py index 991a9890..657a1cff 100755 --- a/py-json/realm.py +++ b/py-json/realm.py @@ -1209,20 +1209,13 @@ class L3CXProfile(LFCliBase): report_file = report_file.replace(str(output_format),'csv',1) print("Saving rolling data into..." + str(report_file)) - #retrieve compared report if specified - turn into dataframe === under construction === - if compared_report is not None: - supported_formats = ['csv', 'json', 'stata', 'pickle','html'] - for format in supported_formats: - if compared_report.lower() == format: - - pass - #================== Step 1, set column names and header row layer3_cols=[self.replace_special_char(x) for x in layer3_cols] layer3_fields = ",".join(layer3_cols) - header_row=layer3_cols - header_row.insert(0,'Timestamp milliseconds') - header_row.insert(0,'Timestamp') + default_cols=['Timestamp','Timestamp milliseconds epoch','Duration elapsed'] + default_cols.extend(layer3_cols) + header_row=default_cols + #csvwriter.writerow([systeminfo['VersionInfo']['BuildVersion'], script_name, str(arguments)]) @@ -1260,7 +1253,13 @@ class L3CXProfile(LFCliBase): time.sleep(10) # for x in range(0,int(round(iterations,0))): + initial_starttime= datetime.datetime.now() while datetime.datetime.now() < end_time: + t = datetime.datetime.now() + timestamp= t.strftime("%m/%d/%Y %I:%M:%S") + t_to_millisec_epoch= int(self.get_milliseconds(t)) + time_elapsed=int(self.get_seconds(t))-int(self.get_seconds(initial_starttime)) + layer_3_response = self.json_get("/endp/%s?fields=%s" % (created_cx, layer3_fields)) if port_mgr_cols is not None: port_mgr_response=self.json_get("/port/1/1/%s?fields=%s" % (sta_list, port_mgr_fields)) @@ -1278,18 +1277,14 @@ class L3CXProfile(LFCliBase): print("Json port_mgr_response from LANforge... " + str(port_mgr_response)) - t = datetime.datetime.now() - - timestamp= t.strftime("%m/%d/%Y %I:%M:%S") - t_to_millisec_epoch= int(self.get_milliseconds(t)) - + temp_list=[] for endpoint in layer_3_response["endpoint"]: if debug: print("Current endpoint values list... ") print(list(endpoint.values())[0]) temp_endp_values=list(endpoint.values())[0] #dict - temp_list.extend([timestamp,t_to_millisec_epoch]) + temp_list.extend([timestamp,t_to_millisec_epoch,time_elapsed]) current_sta = temp_endp_values['name'] merge={} if port_mgr_cols is not None: @@ -1304,7 +1299,8 @@ class L3CXProfile(LFCliBase): for key in port_mgr_values_dict.keys(): renamed_port_cols['port mgr - ' +key]=port_mgr_values_dict[key] merge.update(renamed_port_cols) - for name in header_row[2:-3]: + print(header_row[3:-3]) + for name in header_row[3:-3]: temp_list.append(merge[name]) csvwriter.writerow(temp_list) temp_list.clear() @@ -1326,11 +1322,13 @@ class L3CXProfile(LFCliBase): #comparison to last report / report inputted if compared_report is not None: - compared_report_completed=self.compare_two_df(pd.read_csv(report_file), pd.read_csv(compared_report)) - - #df to final report file output if necessary - if output_format.lower() != 'csv': - self.df_to_file(dataframe=pd.read_csv(report_file), output_f=output_format, save_path=report_file) + compared_df = self.compare_two_df(dataframe_one=self.file_to_df(report_file), dataframe_two=self.file_to_df(compared_report)) + #append compared df to created one + if output_format.lower() != 'csv': + self.df_to_file(dataframe=pd.read_csv(report_file), output_f=output_format, save_path=report_file) + else: + if output_format.lower() != 'csv': + self.df_to_file(dataframe=pd.read_csv(report_file), output_f=output_format, save_path=report_file) def refresh_cx(self): diff --git a/py-scripts/test_ipv4_variable_time.py b/py-scripts/test_ipv4_variable_time.py index 71195561..7addebba 100755 --- a/py-scripts/test_ipv4_variable_time.py +++ b/py-scripts/test_ipv4_variable_time.py @@ -311,14 +311,12 @@ python3 ./test_ipv4_variable_time.py # Retrieve last data file compared_rept = None if args.compared_report: - # check if last report format is same as current rpt format - last_report_format = args.compared_report.split('.')[-1] - if output == last_report_format: - compared_rept = args.compared_report - else: - ValueError( - "Compared report format is not the same as the new report format. Please make sure they are of the same file type.") - + compared_report_format=args.compared_report.split('.')[-1] + #if compared_report_format not in ['csv', 'json', 'dta', 'pkl','html','xlsx','parquet','h5']: + if compared_report_format != 'csv': + ValueError("Cannot process this file type. Please select a different file and re-run script.") + exit(1) + station_list = LFUtils.portNameSeries(prefix_="sta", start_id_=0, end_id_=num_sta - 1, padding_number_=10000, radio=args.radio) ip_var_test = IPV4VariableTime(host=args.mgr, @@ -362,6 +360,7 @@ python3 ./test_ipv4_variable_time.py # send col names here to file to reformat else: port_mgr_cols = args.port_mgr_cols + # send col names here to file to reformat if args.debug: print("Layer 3 Endp column names are...") print(layer3_cols)