From 9db0c32b386332bda715c08bf3ada58a4b1f9b55 Mon Sep 17 00:00:00 2001 From: Dipti Date: Wed, 17 Feb 2021 01:36:24 -0800 Subject: [PATCH] df to file fully implemented and working Signed-off-by: Dipti --- py-json/LANforge/lfcli_base.py | 49 +++++++-------------------- py-json/realm.py | 6 ++-- py-scripts/test_ipv4_variable_time.py | 2 +- 3 files changed, 15 insertions(+), 42 deletions(-) diff --git a/py-json/LANforge/lfcli_base.py b/py-json/LANforge/lfcli_base.py index 12aa3237..6233773a 100644 --- a/py-json/LANforge/lfcli_base.py +++ b/py-json/LANforge/lfcli_base.py @@ -16,6 +16,8 @@ from LANforge import LFRequest import LANforge.LFRequest import csv import pandas as pd +import tables +import pyarrow as pa #import xlsxwriter class LFCliBase: @@ -599,55 +601,28 @@ class LFCliBase: abgnAX : 12, bgnAX : 13 """ - # def write_to_csv_file(self,new_data_list=None, num_cols=0, csvwriter=None,debug=False): - # if num_cols == 0: - # raise ValueError("Number of columns cannot be zero. Exiting script.") - # exit(1) - # if type(new_data_list) is not list: - # raise ValueError("Data type passed to write into CSV is not a list. Exiting script.") - # exit(1) - # if debug: - # print("Writing data : ") - # print(new_data_list) - # indiv_data_line=[] - # col_counter_per_line=0 - # for data_value in new_data_list: - # if col_counter_per_line >= num_cols: - # if debug: - # print(indiv_data_line) - # csvwriter.writerow(indiv_data_line) - # col_counter_per_line=0 - # indiv_data_line.clear() - # indiv_data_line.append(data_value) - # col_counter_per_line += 1 - #================ Pandas Dataframe Functions ====================================== #takes any dataframe and returns the specified file extension of it def df_to_file(self, output_f=None,dataframe=None, save_path=None): - df = dataframe - #pd.set_option("display.max_rows", None, "display.max_columns", None) - #print(df) - if output_f.lower() == 'hdf': - df.to_hdf(save_path, 'table', append=True) + if output_f.lower() == 'hdf': + dataframe.to_hdf(save_path.replace('csv','h5',1), 'table', append=True) if output_f.lower() == 'parquet': - df.to_parquet(save_path, engine='pyarrow') + dataframe.to_parquet(save_path.replace('csv','parquet',1), engine='pyarrow') if output_f.lower() == 'png': - fig = df.plot().get_figure() - fig.savefig(save_path) + fig = dataframe.plot().get_figure() + fig.savefig(save_path.replace('csv','png',1)) if output_f.lower() == 'xlsx': - df.to_excel(save_path) - if output_f.lower() == 'df': - df + dataframe.to_excel(save_path.replace('csv','xlsx',1)) if output_f.lower() == 'json': - df.to_json(save_path) + dataframe.to_json(save_path.replace('csv','json',1)) if output_f.lower() == 'stata': - df.to_stata(save_path) + dataframe.to_stata(save_path.replace('csv','dta',1)) if output_f.lower() == 'pickle': - df.to_pickle(save_path) + dataframe.to_pickle(save_path.replace('csv','pkl',1)) if output_f.lower() == 'html': - df.to_html(save_path) + dataframe.to_html(save_path.replace('csv','html',1)) #takes any format of a file and returns a dataframe of it def file_to_df(self,file_name): diff --git a/py-json/realm.py b/py-json/realm.py index 154733f7..991a9890 100755 --- a/py-json/realm.py +++ b/py-json/realm.py @@ -1214,8 +1214,7 @@ class L3CXProfile(LFCliBase): supported_formats = ['csv', 'json', 'stata', 'pickle','html'] for format in supported_formats: if compared_report.lower() == format: - #exec('df.to_' + x + '("' + report_file + '",index=False' + ')') - #previous_data_df= read_csv() + pass #================== Step 1, set column names and header row @@ -1305,7 +1304,6 @@ class L3CXProfile(LFCliBase): for key in port_mgr_values_dict.keys(): renamed_port_cols['port mgr - ' +key]=port_mgr_values_dict[key] merge.update(renamed_port_cols) - for name in header_row[2:-3]: temp_list.append(merge[name]) csvwriter.writerow(temp_list) @@ -1332,7 +1330,7 @@ class L3CXProfile(LFCliBase): #df to final report file output if necessary if output_format.lower() != 'csv': - self.df_to_file(dataframe=pd.read_csv(report_file), output_f=output_format, save_path=report_file.replace('csv',output_format,1)) + self.df_to_file(dataframe=pd.read_csv(report_file), output_f=output_format, save_path=report_file) def refresh_cx(self): diff --git a/py-scripts/test_ipv4_variable_time.py b/py-scripts/test_ipv4_variable_time.py index 85e5843d..71195561 100755 --- a/py-scripts/test_ipv4_variable_time.py +++ b/py-scripts/test_ipv4_variable_time.py @@ -291,7 +291,7 @@ python3 ./test_ipv4_variable_time.py path = os.path.join(curr_dir_path, new_file_path) os.mkdir(path) - if args.output_format in ['csv', 'json', 'html', 'hdf', 'stata', 'pickle', 'pdf', 'png', 'df', 'parquet', + if args.output_format in ['csv', 'json', 'html', 'hdf','stata', 'pickle', 'pdf', 'png', 'parquet', 'xlsx']: report_f = str(path) + '/data.' + args.output_format output = args.output_format