From 535f15e6c31f55b129cbf552ab3513c0318384b2 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Wed, 16 Dec 2020 23:09:11 -0800 Subject: [PATCH 01/35] lfcli_base.py adds back --security option --- py-json/LANforge/lfcli_base.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/py-json/LANforge/lfcli_base.py b/py-json/LANforge/lfcli_base.py index e35b0b46..a763e779 100644 --- a/py-json/LANforge/lfcli_base.py +++ b/py-json/LANforge/lfcli_base.py @@ -431,14 +431,18 @@ class LFCliBase: return parser - #Create argparse with radio, securiy, ssid and passwd required + # Create argparse with radio, securiy, ssid and passwd required + # TODO: show example of how to add required or optional arguments from calling class @staticmethod def create_basic_argparse(prog=None, formatter_class=None, epilog=None, description=None): if (prog is not None) or (formatter_class is not None) or (epilog is not None) or (description is not None): - parser = argparse.ArgumentParser(prog=prog, formatter_class=formatter_class, epilog=epilog, + parser = argparse.ArgumentParser(prog=prog, + formatter_class=formatter_class, + epilog=epilog, description=description) else: parser = argparse.ArgumentParser() + optional = parser.add_argument_group('optional arguments') required = parser.add_argument_group('required arguments') #Optional Args @@ -452,6 +456,7 @@ class LFCliBase: optional.add_argument('--debug', help='Enable debugging', default=False, action="store_true") #Required Args required.add_argument('--radio', help='radio EID, e.g: 1.wiphy2', required=True) + required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True) required.add_argument('--ssid', help='SSID for stations to associate to', required=True) required.add_argument('--passwd', '--password' ,'--key', help='WiFi passphrase/password/key', required=True) From 2bc6a2a541959dbddfb9d43a67217f4bc172ff13 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Wed, 16 Dec 2020 23:13:08 -0800 Subject: [PATCH 02/35] test_ipv4_variable_time.py: multiple fixes: - whitespace - calls super()._init_ with _local_realm parameter - fixes missing mode parameter check - updates optional and required arguments - adds help for --mode --- py-scripts/test_ipv4_variable_time.py | 123 ++++++++++++++++++-------- 1 file changed, 87 insertions(+), 36 deletions(-) diff --git a/py-scripts/test_ipv4_variable_time.py b/py-scripts/test_ipv4_variable_time.py index 26a08ab3..5f264f83 100755 --- a/py-scripts/test_ipv4_variable_time.py +++ b/py-scripts/test_ipv4_variable_time.py @@ -10,6 +10,7 @@ if sys.version_info[0] != 3: if 'py-json' not in sys.path: sys.path.append(os.path.join(os.path.abspath('..'), 'py-json')) +import pprint import argparse from LANforge.lfcli_base import LFCliBase from LANforge import LFUtils @@ -20,15 +21,35 @@ import datetime class IPV4VariableTime(LFCliBase): def __init__(self, - ssid, security, password, sta_list, name_prefix, upstream, radio, - host="localhost", port=8080, mode = 0, ap=None, - side_a_min_rate=56, side_a_max_rate=0, - side_b_min_rate=56, side_b_max_rate=0, - number_template="00000", test_duration="5m", use_ht160=False, + ssid, + security, + password, + sta_list, + name_prefix, + upstream, + radio, + host="localhost", + port=8080, + mode=0, + ap=None, + side_a_min_rate=56000, + side_a_max_rate=0, # indicates 'same' + side_b_min_rate=56000, + side_b_max_rate=0, # indicates 'same' + number_template="00000", + test_duration="5m", + use_ht160=False, _debug_on=False, _exit_on_error=False, _exit_on_fail=False): - super().__init__(host, port, _debug=_debug_on, _halt_on_error=_exit_on_error, _exit_on_fail=_exit_on_fail) + super().__init__(host, port, + _local_realm = realm.Realm(lfclient_host=host, + lfclient_port=port, + debug_=_debug_on, + halt_on_error_=_exit_on_error), + _debug=_debug_on, + _halt_on_error=_exit_on_error, + _exit_on_fail=_exit_on_fail) self.upstream = upstream self.host = host self.port = port @@ -43,10 +64,8 @@ class IPV4VariableTime(LFCliBase): self.debug = _debug_on self.name_prefix = name_prefix self.test_duration = test_duration - self.local_realm = realm.Realm(lfclient_host=self.host, lfclient_port=self.port) self.station_profile = self.local_realm.new_station_profile() self.cx_profile = self.local_realm.new_l3_cx_profile() - self.station_profile.lfclient_url = self.lfclient_url self.station_profile.ssid = self.ssid self.station_profile.ssid_pass = self.password @@ -56,10 +75,10 @@ class IPV4VariableTime(LFCliBase): self.station_profile.use_ht160 = use_ht160 if self.station_profile.use_ht160: self.station_profile.mode = 9 - self.station_profile.mode = mode + if mode is not None: + self.station_profile.mode = mode if self.ap is not None: - self.station_profile.set_command_param("add_sta", "ap",self.ap) - + self.station_profile.set_command_param("add_sta", "ap", self.ap) self.cx_profile.host = self.host self.cx_profile.port = self.port @@ -69,7 +88,6 @@ class IPV4VariableTime(LFCliBase): self.cx_profile.side_b_min_bps = side_b_min_rate self.cx_profile.side_b_max_bps = side_b_max_rate - def __get_rx_values(self): cx_list = self.json_get("endp?fields=name,rx+bytes", debug_=self.debug) if self.debug: @@ -80,8 +98,8 @@ class IPV4VariableTime(LFCliBase): if cx_name != 'uri' and cx_name != 'handler': for item, value in cx_name.items(): for value_name, value_rx in value.items(): - if value_name == 'rx bytes' and item in self.cx_profile.created_cx.values(): - cx_rx_map[item] = value_rx + if (value_name == 'rx bytes') and (item in self.cx_profile.created_cx.values()): + cx_rx_map[item] = value_rx return cx_rx_map def __compare_vals(self, old_list, new_list): @@ -140,7 +158,6 @@ class IPV4VariableTime(LFCliBase): if passes == expected_passes: self._pass("PASS: All tests passed") - def stop(self): self.cx_profile.stop_cx() self.station_profile.admin_down() @@ -153,19 +170,24 @@ class IPV4VariableTime(LFCliBase): def cleanup(self): self.cx_profile.cleanup() self.station_profile.cleanup() - LFUtils.wait_until_ports_disappear(base_url=self.lfclient_url, port_list=self.station_profile.station_names, + LFUtils.wait_until_ports_disappear(base_url=self.lfclient_url, + port_list=self.station_profile.station_names, debug=self.debug) def build(self): - self.station_profile.use_security(self.security, self.ssid, self.password) self.station_profile.set_number_template(self.number_template) print("Creating stations") self.station_profile.set_command_flag("add_sta", "create_admin_down", 1) self.station_profile.set_command_param("set_port", "report_timer", 1500) self.station_profile.set_command_flag("set_port", "rpt_timer", 1) - self.station_profile.create(radio=self.radio, sta_names_=self.sta_list, debug=self.debug) - self.cx_profile.create(endp_type="lf_udp", side_a=self.station_profile.station_names, side_b=self.upstream, sleep_time=0) + self.station_profile.create(radio=self.radio, + sta_names_=self.sta_list, + debug=self.debug) + self.cx_profile.create(endp_type="lf_udp", + side_a=self.station_profile.station_names, + side_b=self.upstream, + sleep_time=0) self._pass("PASS: Station build finished") @@ -174,16 +196,13 @@ def main(): prog='test_ipv4_variable_time.py', formatter_class=argparse.RawTextHelpFormatter, epilog='''\ - Create stations to test connection and traffic on VAPs of varying security types (WEP, WPA, WPA2, WPA3, Open) - ''', +Create stations to test connection and traffic on VAPs of varying security types (WEP, WPA, WPA2, WPA3, Open) +''', description='''\ test_ipv4_variable_time.py: -------------------- -Generic command layout: - - python3 ./test_ipv4_variable_time.py - --upstream_port eth1 +Options: --upstream_port eth1 --radio wiphy0 --num_stations 32 --security {open|wep|wpa|wpa2|wpa3} \\ @@ -208,17 +227,46 @@ Generic command layout: --a_min 1000 --b_min 1000 --ap "00:0e:8e:78:e1:76" - --debug - ''') + --debug''') + + #optional = parser.add_argument_group('optional arguments') + #required = parser.add_argument_group('required arguments') + required = None + for agroup in parser._action_groups: + if agroup.title == "required arguments": + required = agroup + # pprint.pprint(required) + if required is not None: + required.add_argument('--a_min', help='bps rate minimum for endpoint A', type=int, default=256000) + required.add_argument('--b_min', help='bps rate minimum for endpoint B', type=int, default=256000) + required.add_argument('--test_duration', help='duration of the test', default="2m") + + optional = None + for agroup in parser._action_groups: + if agroup.title == "optional arguments": + optional = agroup + # pprint.pprint(optional) + if optional is not None: + optional.add_argument('--monitor', help='print CSV output of connection status', default=False, action='store_true') + optional.add_argument('--save_to', help='save CSV monitor output to file', default=None) + optional.add_argument('--monitor_interval_sec', help='seconds between polling endpoints', default=5) + optional.add_argument('--mode' help="""Station WiFi modes: use the number value below: + auto : 0, + a : 1, + b : 2, + g : 3, + abg : 4, + abgn : 5, + bgn : 6, + bg : 7, + abgnAC : 8, + anAC : 9, + an : 10, + bgnAC : 11, + abgnAX : 12, + bgnAX : 13, +""") - optional = parser.add_argument_group('optional arguments') - required = parser.add_argument_group('required arguments') - parser.add_argument('--a_min', help='--a_min bps rate minimum for side_a', default=256000) - parser.add_argument('--b_min', help='--b_min bps rate minimum for side_b', default=256000) - parser.add_argument('--test_duration', help='--test_duration sets the duration of the test', default="2m") - required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True) - optional.add_argument('--mode',help='Used to force mode of stations') - optional.add_argument('--ap',help='Used to force a connection to a particular AP') args = parser.parse_args() num_sta = 2 @@ -226,7 +274,10 @@ Generic command layout: num_sta = int(args.num_stations) station_list = LFUtils.portNameSeries(prefix_="sta", start_id_=0, end_id_=num_sta-1, padding_number_=10000, radio=args.radio) - ip_var_test = IPV4VariableTime(host=args.mgr, port=args.mgr_port, number_template="00", sta_list=station_list, + ip_var_test = IPV4VariableTime(host=args.mgr, + port=args.mgr_port, + number_template="00", + sta_list=station_list, name_prefix="VT", upstream=args.upstream_port, ssid=args.ssid, From c21eb184c4751852cc82d535796fb5243e5876cd Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Wed, 16 Dec 2020 23:18:36 -0800 Subject: [PATCH 03/35] test_ipv4_variable_time.py: fixes comma, adds --ap argument --- py-scripts/test_ipv4_variable_time.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py-scripts/test_ipv4_variable_time.py b/py-scripts/test_ipv4_variable_time.py index 5f264f83..395a061a 100755 --- a/py-scripts/test_ipv4_variable_time.py +++ b/py-scripts/test_ipv4_variable_time.py @@ -250,7 +250,7 @@ Options: --upstream_port eth1 optional.add_argument('--monitor', help='print CSV output of connection status', default=False, action='store_true') optional.add_argument('--save_to', help='save CSV monitor output to file', default=None) optional.add_argument('--monitor_interval_sec', help='seconds between polling endpoints', default=5) - optional.add_argument('--mode' help="""Station WiFi modes: use the number value below: + optional.add_argument('--mode', help="""Station WiFi modes: use the number value below: auto : 0, a : 1, b : 2, @@ -266,6 +266,7 @@ Options: --upstream_port eth1 abgnAX : 12, bgnAX : 13, """) + optional.add_argument("--ap", help="Add BSSID of access point to connect to") args = parser.parse_args() From 6594a6ad34336bb12673b0848b271aeb843a104b Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Wed, 16 Dec 2020 23:47:32 -0800 Subject: [PATCH 04/35] lfcli_base.py: adds constant text for help on mode settings --- py-json/LANforge/lfcli_base.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/py-json/LANforge/lfcli_base.py b/py-json/LANforge/lfcli_base.py index a763e779..1bac0034 100644 --- a/py-json/LANforge/lfcli_base.py +++ b/py-json/LANforge/lfcli_base.py @@ -472,4 +472,21 @@ class LFCliBase: } self.json_post("/cli-json/add_event", data, debug_=debug_) + Help_Mode = """Station WiFi modes: use the number value below: + auto : 0, + a : 1, + b : 2, + g : 3, + abg : 4, + abgn : 5, + bgn : 6, + bg : 7, + abgnAC : 8, + anAC : 9, + an : 10, + bgnAC : 11, + abgnAX : 12, + bgnAX : 13 +""" + # ~class From 35a4e3392cb7f39a0ae484c2959d2e3a7cfca63a Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Wed, 16 Dec 2020 23:49:33 -0800 Subject: [PATCH 05/35] test_ipv4_variable_time.py cleans up very large help section --- py-scripts/test_ipv4_variable_time.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/py-scripts/test_ipv4_variable_time.py b/py-scripts/test_ipv4_variable_time.py index 395a061a..5ed682a3 100755 --- a/py-scripts/test_ipv4_variable_time.py +++ b/py-scripts/test_ipv4_variable_time.py @@ -247,26 +247,11 @@ Options: --upstream_port eth1 optional = agroup # pprint.pprint(optional) if optional is not None: - optional.add_argument('--monitor', help='print CSV output of connection status', default=False, action='store_true') - optional.add_argument('--save_to', help='save CSV monitor output to file', default=None) - optional.add_argument('--monitor_interval_sec', help='seconds between polling endpoints', default=5) - optional.add_argument('--mode', help="""Station WiFi modes: use the number value below: - auto : 0, - a : 1, - b : 2, - g : 3, - abg : 4, - abgn : 5, - bgn : 6, - bg : 7, - abgnAC : 8, - anAC : 9, - an : 10, - bgnAC : 11, - abgnAX : 12, - bgnAX : 13, -""") optional.add_argument("--ap", help="Add BSSID of access point to connect to") + optional.add_argument('--mode', help=LFCliBase.Help_Mode) + optional.add_argument('--monitor', help='print CSV output of connection status', default=False, action='store_true') + optional.add_argument('--monitor_interval_sec', help='seconds between polling endpoints', default=5) + optional.add_argument('--save_to', help='save CSV monitor output to file', default=None) args = parser.parse_args() From bf84f54d58681ff02b0c423b1cebbb84d9d504d3 Mon Sep 17 00:00:00 2001 From: Chuck SmileyRekiere Date: Wed, 16 Dec 2020 16:44:23 -0700 Subject: [PATCH 06/35] lf_cisco_snp.py : updates for results csv --- py-scripts/lf_cisco_snp.py | 111 +++++++++++++++++++++++++------------ 1 file changed, 77 insertions(+), 34 deletions(-) diff --git a/py-scripts/lf_cisco_snp.py b/py-scripts/lf_cisco_snp.py index 85c75981..a9be3978 100755 --- a/py-scripts/lf_cisco_snp.py +++ b/py-scripts/lf_cisco_snp.py @@ -622,11 +622,12 @@ class cisco_(): class L3VariableTime(LFCliBase): def __init__(self, host, port, endp_type, args, tos, side_b, radio_name_list, number_of_stations_per_radio_list, - ssid_list, ssid_password_list, ssid_security_list, wifimode_list,station_lists, name_prefix, debug_on, outfile, + ssid_list, ssid_password_list, ssid_security_list, wifimode_list,station_lists, name_prefix, debug_on, outfile, results, test_keys,test_config, reset_port_enable_list, reset_port_time_min_list, reset_port_time_max_list, + csv_started=False, side_a_min_bps=560000, side_a_max_bps=0, side_a_min_pdu=1518,side_a_max_pdu=0, side_b_min_bps=560000, side_b_max_bps=0, @@ -662,7 +663,8 @@ class L3VariableTime(LFCliBase): self.station_profiles = [] self.args = args self.outfile = outfile - self.csv_started = False + self.results = results + self.csv_started = csv_started self.epoch_time = int(time.time()) self.debug = debug_on self.test_keys = test_keys @@ -676,6 +678,10 @@ class L3VariableTime(LFCliBase): self.csv_file = open(self.outfile, "a+") self.csv_writer = csv.writer(self.csv_file, delimiter=",") + if self.results is not None: + self.csv_results = open(self.results, "a+") + self.csv_results_writer = csv.writer(self.csv_results, delimiter=",") + for (radio_, ssid_, ssid_password_, ssid_security_, wifimode_,\ reset_port_enable_, reset_port_time_min_, reset_port_time_max_) \ in zip(radio_name_list, ssid_list, ssid_password_list, ssid_security_list, wifimode_list,\ @@ -766,6 +772,7 @@ class L3VariableTime(LFCliBase): csv_rx_drop_percent_data.append(rx_drop_percent[item]) self.csv_add_row(csv_rx_drop_percent_data,self.csv_writer,self.csv_file) + self.csv_add_row(csv_rx_drop_percent_data,self.csv_results_writer,self.csv_results) def __compare_vals(self, old_list, new_list): passes = 0 @@ -776,9 +783,9 @@ class L3VariableTime(LFCliBase): csv_rx_delta_row_data = [] csv_rx_delta_dict = {} - for key in self.test_keys: - csv_rx_row_data.append(self.test_config_dict[key]) - csv_rx_delta_row_data.append(self.test_config_dict[key]) + #for key in self.test_keys: + # csv_rx_row_data.append(self.test_config_dict[key]) + # csv_rx_delta_row_data.append(self.test_config_dict[key]) for key in [key for key in old_list if "mtx" in key]: del old_list[key] @@ -787,8 +794,6 @@ class L3VariableTime(LFCliBase): filtered_values = [v for _, v in new_list.items() if v !=0] average_rx= sum(filtered_values) / len(filtered_values) if len(filtered_values) != 0 else 0 - - # only evaluate upstream or downstream new_evaluate_list = new_list.copy() print("new_evaluate_list before",new_evaluate_list) @@ -798,6 +803,7 @@ class L3VariableTime(LFCliBase): elif "downstream" in self.test_config_dict.values(): for key in [key for key in new_evaluate_list if "-B" in key]: del new_evaluate_list[key] print("downstream in dictionary values") + #follow code left in for now provides the best 5 worst 5 '''print("new_evaluate_list after",new_evaluate_list) csv_performance_values=sorted(new_evaluate_list.items(), key=lambda x: (x[1],x[0]), reverse=False) csv_performance_values=self.csv_validate_list(csv_performance_values,5) @@ -819,6 +825,7 @@ class L3VariableTime(LFCliBase): if len(old_evaluate_list) == len(new_evaluate_list): for item, value in old_evaluate_list.items(): expected_passes +=1 + print("ITEM: {} VALUE: {}".format(item, value)) if new_evaluate_list[item] > old_evaluate_list[item]: passes += 1 #if self.debug: logg.info(item, new_evaluate_list[item], old_evaluate_list[item], " Difference: ", new_evaluate_list[item] - old_evaluate_list[item]) @@ -840,6 +847,9 @@ class L3VariableTime(LFCliBase): # need to generate list first to determine worst and best filtered_values = [v for _, v in csv_rx_delta_dict.items() if v !=0] #average_rx_delta= sum(filtered_values) / len(filtered_values) if len(filtered_values) != 0 else 0 + for key in self.test_keys: + csv_rx_row_data.append(self.test_config_dict[key]) + csv_rx_delta_row_data.append(self.test_config_dict[key]) max_tp_mbps = sum(filtered_values) csv_rx_row_data.append(max_tp_mbps) @@ -855,6 +865,9 @@ class L3VariableTime(LFCliBase): csv_rx_row_data.append("fail") csv_rx_row_data.extend([self.epoch_time, self.time_stamp(),'rx_delta']) + + print("csv_rx_row_data {}".format(csv_rx_row_data)) + #TODO: may want to pass in the information that needs to be in the csv file into the class ''' csv_rx_row_data.extend([self.epoch_time, self.time_stamp(),'rx']) csv_rx_delta_row_data.extend([self.epoch_time, self.time_stamp(),'rx_delta']) @@ -880,20 +893,23 @@ class L3VariableTime(LFCliBase): csv_rx_headers.append(item) # note need to have all upstream and downstream in the csv table thus new_list and old_list #csv_rx_row_data.append(new_list[item]) + # provide delta csv_rx_row_data.append(new_list[item] - old_list[item]) self.csv_add_row(csv_rx_row_data,self.csv_writer,self.csv_file) + #self.csv_add_row(csv_rx_row_data,self.csv_results_writer,self.csv_results) + #self.csv_add_row(csv_rx_delta_row_data,self.csv_writer,self.csv_file) if passes == expected_passes: - return True + return True, max_tp_mbps, csv_rx_row_data else: - return False + return False, max_tp_mbps, csv_rx_row_data else: print("Old-list length: %i new: %i does not match in compare-vals."%(len(old_list), len(new_list))) print("old-list:",old_list) print("new-list:",new_list) - return False + return False, None, None # check to see if this is valid def verify_controller(self): if self.args == None: @@ -1002,6 +1018,11 @@ class L3VariableTime(LFCliBase): self._pass("PASS: Stations build finished") def start(self, print_pass=False, print_fail=False): + best_max_tp_mbps = 0 + best_csv_rx_row_data = " " + max_tp_mbps = 0 + csv_rx_row_data = " " + Result = False logg.info("Bringing up stations") self.local_realm.admin_up(self.side_b) for station_profile in self.station_profiles: @@ -1052,7 +1073,21 @@ class L3VariableTime(LFCliBase): new_rx_values, rx_drop_percent = self.__get_rx_values() expected_passes += 1 - if self.__compare_vals(old_rx_values, new_rx_values): + ''' + #self.csv_add_row(csv_rx_row_data,self.csv_results_writer,self.csv_results) + + + if passes == expected_passes: + return True, max_tp_mbps, csv_rx_row_data + else: + return False, max_tp_mbps, csv_rx_row_data + ''' + Result, max_tp_mbps, csv_rx_row_data = self.__compare_vals(old_rx_values, new_rx_values) + if max_tp_mbps > best_max_tp_mbps: + best_max_tp_mbps = max_tp_mbps + best_csv_rx_row_data = csv_rx_row_data + + if Result: passes += 1 else: self._fail("FAIL: Not all stations increased traffic", print_fail) @@ -1062,7 +1097,7 @@ class L3VariableTime(LFCliBase): #self.__record_rx_dropped_percent(rx_drop_percent) cur_time = datetime.datetime.now() - + self.csv_add_row(best_csv_rx_row_data,self.csv_results_writer,self.csv_results) if passes == expected_passes: self._pass("PASS: All tests passed", print_pass) @@ -1095,14 +1130,17 @@ class L3VariableTime(LFCliBase): if self.csv_file is not None: self.csv_writer.writerow(headers) self.csv_file.flush() + if self.csv_results is not None: + self.csv_results_writer.writerow(headers) + self.csv_results.flush() def csv_validate_list(self, csv_list, length): if len(csv_list) < length: csv_list = csv_list + [('no data','no data')] * (length - len(csv_list)) return csv_list - def csv_add_row(self,row,writer,csv_file): - if self.csv_file is not None: + def csv_add_row(self,row,writer,csv_file): # can make two calls eventually + if csv_file is not None: writer.writerow(row) csv_file.flush() @@ -1466,6 +1504,7 @@ Eventual Realm at Cisco if args.csv_outfile != None: current_time = time.strftime("%m_%d_%Y_%H_%M_%S", time.localtime()) csv_outfile = "{}_{}.csv".format(args.csv_outfile,current_time) + csv_results = "results_{}_{}.csv".format(args.csv_outfile,current_time) print("csv output file : {}".format(csv_outfile)) if args.log: @@ -1707,12 +1746,12 @@ Eventual Realm at Cisco cisco_chan_widths = "20".split() cisco_ap_modes = "local".split() cisco_data_encryptions = "disable".split() - cisco_packet_types = "lf_udp lf_tcp".split() - #cisco_packet_types = "lf_udp".split() + #cisco_packet_types = "lf_udp lf_tcp".split() + cisco_packet_types = "lf_udp".split() #cisco_directions = "upstream downstream".split() cisco_directions = "upstream downstream".split() - cisco_packet_sizes = "88 512 1370 1518".split() - #cisco_packet_sizes = "1518".split() + #cisco_packet_sizes = "88 512 1370 1518".split() + cisco_packet_sizes = "1518".split() cisco_client_densities = "1".split() cisco_data_encryptions = "disable".split() @@ -1760,11 +1799,12 @@ Eventual Realm at Cisco logg.info(cisco_client_densities) logg.info(cisco_data_encryptions) - ap_set = None - band_set = None - chan_width_set = None - ap_mode_set = None - tx_power_set = None + __ap_set = None + __band_set = None + __chan_width_set = None + __ap_mode_set = None + __tx_power_set = None + __csv_started = False for cisco_ap in cisco_aps: for cisco_band in cisco_bands: # frequency @@ -1794,20 +1834,20 @@ Eventual Realm at Cisco logg.info("# NO CONTROLLER SET , TEST MODE") logg.info("################################################") else: - if( cisco_ap != ap_set or - cisco_band != band_set or - cisco_chan_width != chan_width_set or - cisco_ap_mode != ap_mode_set or - cisco_tx_power != tx_power_set + if( cisco_ap != __ap_set or + cisco_band != __band_set or + cisco_chan_width != __chan_width_set or + cisco_ap_mode != __ap_mode_set or + cisco_tx_power != __tx_power_set ): logg.info("###############################################") logg.info("# NEW CONTROLLER CONFIG") logg.info("###############################################") - ap_set = cisco_ap - band_set = cisco_band - chan_width_set = cisco_chan_width - ap_mode_set = cisco_ap_mode - tx_power_set = cisco_tx_power + __ap_set = cisco_ap + __band_set = cisco_band + __chan_width_set = cisco_chan_width + __ap_mode_set = cisco_ap_mode + __tx_power_set = cisco_tx_power ############################################# # configure cisco controller ############################################# @@ -1969,8 +2009,11 @@ Eventual Realm at Cisco side_b_min_pdu =cisco_packet_size, debug_on=debug_on, outfile=csv_outfile, + results=csv_results, test_keys=test_keys, - test_config=test_config) + test_config=test_config, + csv_started=__csv_started ) + __csv_started = True ip_var_test.pre_cleanup() ip_var_test.build() if not ip_var_test.passes(): From 33eabbc61aae962975d795d622360c30291e3711 Mon Sep 17 00:00:00 2001 From: Chuck SmileyRekiere Date: Thu, 17 Dec 2020 07:10:35 -0700 Subject: [PATCH 07/35] lf_cisco_snp.py : created results csv , added test_id --- py-scripts/lf_cisco_snp.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/py-scripts/lf_cisco_snp.py b/py-scripts/lf_cisco_snp.py index a9be3978..13d593d1 100755 --- a/py-scripts/lf_cisco_snp.py +++ b/py-scripts/lf_cisco_snp.py @@ -782,6 +782,7 @@ class L3VariableTime(LFCliBase): csv_rx_row_data = [] csv_rx_delta_row_data = [] csv_rx_delta_dict = {} + test_id = "" #for key in self.test_keys: # csv_rx_row_data.append(self.test_config_dict[key]) @@ -803,7 +804,7 @@ class L3VariableTime(LFCliBase): elif "downstream" in self.test_config_dict.values(): for key in [key for key in new_evaluate_list if "-B" in key]: del new_evaluate_list[key] print("downstream in dictionary values") - #follow code left in for now provides the best 5 worst 5 + #follow code left in for now, provides the best 5 worst 5 '''print("new_evaluate_list after",new_evaluate_list) csv_performance_values=sorted(new_evaluate_list.items(), key=lambda x: (x[1],x[0]), reverse=False) csv_performance_values=self.csv_validate_list(csv_performance_values,5) @@ -850,6 +851,7 @@ class L3VariableTime(LFCliBase): for key in self.test_keys: csv_rx_row_data.append(self.test_config_dict[key]) csv_rx_delta_row_data.append(self.test_config_dict[key]) + max_tp_mbps = sum(filtered_values) csv_rx_row_data.append(max_tp_mbps) @@ -858,6 +860,13 @@ class L3VariableTime(LFCliBase): expected_tp_mbps = max_tp_mbps csv_rx_row_data.append(expected_tp_mbps) + #Generate TestID + for key in self.test_keys: + test_id = test_id + "_" + self.test_config_dict[key] + + print("test_id: {}".format(test_id)) + csv_rx_row_data.append(test_id) + # Todo pass or fail if max_tp_mbps == expected_tp_mbps: csv_rx_row_data.append("pass") @@ -1082,11 +1091,13 @@ class L3VariableTime(LFCliBase): else: return False, max_tp_mbps, csv_rx_row_data ''' + # __compare_vals - does the calculations Result, max_tp_mbps, csv_rx_row_data = self.__compare_vals(old_rx_values, new_rx_values) if max_tp_mbps > best_max_tp_mbps: best_max_tp_mbps = max_tp_mbps best_csv_rx_row_data = csv_rx_row_data + # need to check the expected max_tp_mbps if Result: passes += 1 else: @@ -1118,7 +1129,7 @@ class L3VariableTime(LFCliBase): def csv_generate_column_headers(self): csv_rx_headers = self.test_keys.copy() csv_rx_headers.extend - csv_rx_headers.extend(['Max TP Mbps','Expected TP','Pass Fail','Time epoch','Time','Monitor']) + csv_rx_headers.extend(['max_tp_mbps','expected_tp','test_id','pass_fail','epoch_time','time','monitor']) '''for i in range(1,6): csv_rx_headers.append("least_rx_data {}".format(i)) for i in range(1,6): @@ -1828,6 +1839,7 @@ Eventual Realm at Cisco cisco_band,cisco_wifimode,cisco_chan_width,cisco_data_encryption,cisco_ap_mode,cisco_client_density, cisco_packet_type,cisco_direction,cisco_packet_size) test_keys = ['AP','Band','wifi_mode','BW','encryption','ap_mode','clients','packet_type','direction','packet_size'] + logg.info("# Cisco run settings: {}".format(test_config)) if(args.no_controller): logg.info("################################################") From eef7eb1c76f7dd5066c6d0b0a5c674f49d1f5950 Mon Sep 17 00:00:00 2001 From: Chuck SmileyRekiere Date: Thu, 17 Dec 2020 15:41:36 -0700 Subject: [PATCH 08/35] lf_cisco_snp.py : clean up monitoring summary --- py-scripts/lf_cisco_snp.py | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/py-scripts/lf_cisco_snp.py b/py-scripts/lf_cisco_snp.py index 13d593d1..9d76f9a6 100755 --- a/py-scripts/lf_cisco_snp.py +++ b/py-scripts/lf_cisco_snp.py @@ -780,6 +780,7 @@ class L3VariableTime(LFCliBase): csv_performance_values = [] csv_rx_headers = [] csv_rx_row_data = [] + csv_result_row_data = [] csv_rx_delta_row_data = [] csv_rx_delta_dict = {} test_id = "" @@ -843,6 +844,13 @@ class L3VariableTime(LFCliBase): csv_header += csv_rx_headers logg.info(csv_header) self.csv_add_column_headers(csv_header) + csv_results = self.csv_generate_column_results_headers() + #csv_results += csv_rx_headers + self.csv_add_column_headers_results(csv_results) + print("###################################") + print(csv_results) + print("###################################") + self.csv_started = True # need to generate list first to determine worst and best @@ -850,15 +858,18 @@ class L3VariableTime(LFCliBase): #average_rx_delta= sum(filtered_values) / len(filtered_values) if len(filtered_values) != 0 else 0 for key in self.test_keys: csv_rx_row_data.append(self.test_config_dict[key]) + csv_result_row_data.append(self.test_config_dict[key]) csv_rx_delta_row_data.append(self.test_config_dict[key]) - max_tp_mbps = sum(filtered_values) csv_rx_row_data.append(max_tp_mbps) + csv_result_row_data.append(max_tp_mbps) #To do needs to be read or passed in based on test type expected_tp_mbps = max_tp_mbps csv_rx_row_data.append(expected_tp_mbps) + csv_result_row_data.append(expected_tp_mbps) + #Generate TestID for key in self.test_keys: @@ -866,14 +877,18 @@ class L3VariableTime(LFCliBase): print("test_id: {}".format(test_id)) csv_rx_row_data.append(test_id) + csv_result_row_data.append(test_id) # Todo pass or fail if max_tp_mbps == expected_tp_mbps: csv_rx_row_data.append("pass") + csv_result_row_data.append("pass") else: csv_rx_row_data.append("fail") + csv_result_row_data.append("fail") csv_rx_row_data.extend([self.epoch_time, self.time_stamp(),'rx_delta']) + csv_result_row_data.extend([self.epoch_time, self.time_stamp()]) print("csv_rx_row_data {}".format(csv_rx_row_data)) #TODO: may want to pass in the information that needs to be in the csv file into the class @@ -911,9 +926,9 @@ class L3VariableTime(LFCliBase): #self.csv_add_row(csv_rx_delta_row_data,self.csv_writer,self.csv_file) if passes == expected_passes: - return True, max_tp_mbps, csv_rx_row_data + return True, max_tp_mbps, csv_result_row_data else: - return False, max_tp_mbps, csv_rx_row_data + return False, max_tp_mbps, csv_result_row_data else: print("Old-list length: %i new: %i does not match in compare-vals."%(len(old_list), len(new_list))) print("old-list:",old_list) @@ -1137,10 +1152,24 @@ class L3VariableTime(LFCliBase): csv_rx_headers.append("average_rx_data")''' return csv_rx_headers + def csv_generate_column_results_headers(self): + csv_rx_headers = self.test_keys.copy() + csv_rx_headers.extend + csv_rx_headers.extend(['max_tp_mbps','expected_tp','test_id','pass_fail','epoch_time','time']) + '''for i in range(1,6): + csv_rx_headers.append("least_rx_data {}".format(i)) + for i in range(1,6): + csv_rx_headers.append("most_rx_data_{}".format(i)) + csv_rx_headers.append("average_rx_data")''' + return csv_rx_headers + + def csv_add_column_headers(self,headers): if self.csv_file is not None: self.csv_writer.writerow(headers) self.csv_file.flush() + + def csv_add_column_headers_results(self,headers): if self.csv_results is not None: self.csv_results_writer.writerow(headers) self.csv_results.flush() @@ -1763,7 +1792,7 @@ Eventual Realm at Cisco cisco_directions = "upstream downstream".split() #cisco_packet_sizes = "88 512 1370 1518".split() cisco_packet_sizes = "1518".split() - cisco_client_densities = "1".split() + cisco_client_densities = "10".split() cisco_data_encryptions = "disable".split() cisco_side_a_min_bps = 500000000 From 4edf9cb73cde76121c94087253721f795a19d1ed Mon Sep 17 00:00:00 2001 From: Logan Lipke Date: Thu, 17 Dec 2020 15:32:48 -0800 Subject: [PATCH 09/35] FIOEndpProfile create can now take a connections_per_port arg --- py-json/realm.py | 105 ++++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/py-json/realm.py b/py-json/realm.py index 88c1f9ca..76a1cb8a 100755 --- a/py-json/realm.py +++ b/py-json/realm.py @@ -2452,68 +2452,69 @@ class FIOEndpProfile(LFCliBase): self.json_post(req_url, data) #pprint(data) - def create(self, ports=[], sleep_time=.5, debug_=False, suppress_related_commands_=None): + def create(self, ports=[], connections_per_port=1, sleep_time=.5, debug_=False, suppress_related_commands_=None): cx_post_data = [] for port_name in ports: - if len(self.local_realm.name_to_eid(port_name)) == 3: - shelf = self.local_realm.name_to_eid(port_name)[0] - resource = self.local_realm.name_to_eid(port_name)[1] - name = self.local_realm.name_to_eid(port_name)[2] - else: - raise ValueError("Unexpected name for port_name %s" % port_name) - if self.directory is None or self.server_mount is None or self.fs_type is None: - raise ValueError("directory [%s], server_mount [%s], and type [%s] must not be None" % (self.directory, self.server_mount, self.fs_type)) - endp_data = { - "alias": self.cx_prefix + name + "_fio", - "shelf": shelf, - "resource": resource, - "port": name, - "type": self.fs_type, - "min_read_rate": self.min_read_rate_bps, - "max_read_rate": self.max_read_rate_bps, - "min_write_rate": self.min_write_rate_bps, - "max_write_rate": self.max_write_rate_bps, - "directory": self.directory, - "server_mount": self.server_mount, - "mount_dir": self.mount_dir, - "prefix": self.file_prefix, - "payload_pattern": self.pattern, + for num_connection in range(connections_per_port): + if len(self.local_realm.name_to_eid(port_name)) == 3: + shelf = self.local_realm.name_to_eid(port_name)[0] + resource = self.local_realm.name_to_eid(port_name)[1] + name = self.local_realm.name_to_eid(port_name)[2] + else: + raise ValueError("Unexpected name for port_name %s" % port_name) + if self.directory is None or self.server_mount is None or self.fs_type is None: + raise ValueError("directory [%s], server_mount [%s], and type [%s] must not be None" % (self.directory, self.server_mount, self.fs_type)) + endp_data = { + "alias": self.cx_prefix + name + "_" + str(num_connection) + "_fio" , + "shelf": shelf, + "resource": resource, + "port": name, + "type": self.fs_type, + "min_read_rate": self.min_read_rate_bps, + "max_read_rate": self.max_read_rate_bps, + "min_write_rate": self.min_write_rate_bps, + "max_write_rate": self.max_write_rate_bps, + "directory": self.directory, + "server_mount": self.server_mount, + "mount_dir": self.mount_dir, + "prefix": self.file_prefix, + "payload_pattern": self.pattern, - } - # Read direction is copy of write only directory - if self.io_direction == "read": - endp_data["prefix"] = "wo_" + name + "_fio" - endp_data["directory"] = "/mnt/lf/wo_" + name + "_fio" + } + # Read direction is copy of write only directory + if self.io_direction == "read": + endp_data["prefix"] = "wo_" + name + "_" + str(num_connection) + "_fio" + endp_data["directory"] = "/mnt/lf/wo_" + name + "_" + str(num_connection) + "_fio" - url = "cli-json/add_file_endp" - self.local_realm.json_post(url, endp_data, debug_=True, suppress_related_commands_=suppress_related_commands_) - time.sleep(sleep_time) + url = "cli-json/add_file_endp" + self.local_realm.json_post(url, endp_data, debug_=True, suppress_related_commands_=suppress_related_commands_) + time.sleep(sleep_time) - data = { - "name": self.cx_prefix + name + "_fio", - "io_direction": self.io_direction, - "num_files": 5 - } - self.local_realm.json_post("cli-json/set_fe_info", data, debug_=debug_, - suppress_related_commands_=suppress_related_commands_) + data = { + "name": self.cx_prefix + name + "_" + str(num_connection) + "_fio" , + "io_direction": self.io_direction, + "num_files": 5 + } + self.local_realm.json_post("cli-json/set_fe_info", data, debug_=debug_, + suppress_related_commands_=suppress_related_commands_) self.local_realm.json_post("/cli-json/nc_show_endpoints", {"endpoint": "all"}) for port_name in ports: - if len(self.local_realm.name_to_eid(port_name)) == 3: - shelf = self.local_realm.name_to_eid(port_name)[0] - resource = self.local_realm.name_to_eid(port_name)[1] - name = self.local_realm.name_to_eid(port_name)[2] + for num_connection in range(connections_per_port): + if len(self.local_realm.name_to_eid(port_name)) == 3: + shelf = self.local_realm.name_to_eid(port_name)[0] + resource = self.local_realm.name_to_eid(port_name)[1] + name = self.local_realm.name_to_eid(port_name)[2] - endp_data = { - "alias": "CX_" + self.cx_prefix + name + "_fio", - "test_mgr": "default_tm", - "tx_endp": self.cx_prefix + name + "_fio", - "rx_endp": "NA" - } - cx_post_data.append(endp_data) - self.created_cx[self.cx_prefix + name + "_fio"] = "CX_" + self.cx_prefix + name + "_fio" + endp_data = { + "alias": "CX_" + self.cx_prefix + name + "_" + str(num_connection) + "_fio" , + "test_mgr": "default_tm", + "tx_endp": self.cx_prefix + name + "_" + str(num_connection) + "_fio" , + "rx_endp": "NA" + } + cx_post_data.append(endp_data) + self.created_cx[self.cx_prefix + name + "_" + str(num_connection) + "_fio" ] = "CX_" + self.cx_prefix + name + "_" + str(num_connection) + "_fio" - # time.sleep(3) for cx_data in cx_post_data: url = "/cli-json/add_cx" self.local_realm.json_post(url, cx_data, debug_=debug_, suppress_related_commands_=suppress_related_commands_) From 497838bf6c4ac987a284a7ce1be3f2775d02cd7e Mon Sep 17 00:00:00 2001 From: Logan Lipke Date: Thu, 17 Dec 2020 15:33:21 -0800 Subject: [PATCH 10/35] Multiple connections can be created per port. ips on existing ports can be changed individually --- py-scripts/test_fileio.py | 97 ++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/py-scripts/test_fileio.py b/py-scripts/test_fileio.py index e16c8f00..d969a086 100755 --- a/py-scripts/test_fileio.py +++ b/py-scripts/test_fileio.py @@ -50,6 +50,7 @@ class FileIOTest(LFCliBase): read_only_test_group=None, port_list=[], ip_list=None, + connections_per_port=1, mode="both", update_group_args={"name": None, "action": None, "cxs": None}, _debug_on=False, @@ -66,6 +67,7 @@ class FileIOTest(LFCliBase): self.number_template = number_template self.test_duration = test_duration self.port_list = [] + self.connections_per_port = connections_per_port self.use_macvlans = use_macvlans self.mode = mode.lower() self.ip_list = ip_list @@ -131,6 +133,7 @@ class FileIOTest(LFCliBase): self.wo_profile.max_write_rate_bps = LFUtils.parse_size(max_write_rate_bps) self.wo_profile.directory = directory self.wo_profile.server_mount = server_mount + self.wo_profile.num_connections_per_port = connections_per_port self.ro_profile = self.wo_profile.create_ro_profile() @@ -280,25 +283,26 @@ class FileIOTest(LFCliBase): self.created_ports += self.station_profile.station_names if len(self.ip_list) > 0: - if self.gateway is not None and self.netmask is not None: - print("++++++++++++++++\n", self.ip_list, "++++++++++++++++\n") - for num_port in range(len(self.port_list)): - shelf = self.local_realm.name_to_eid(self.port_list[num_port])[0] - resource = self.local_realm.name_to_eid(self.port_list[num_port])[1] - port = self.local_realm.name_to_eid(self.port_list[num_port])[2] - req_url = "/cli-json/set_port" - data = { - "shelf": shelf, - "resource": resource, - "port": port, - "ip_addr": self.ip_list[num_port], - "netmask": self.netmask, - "gateway": self.gateway - } - self.local_realm.json_post(req_url, data) - self.created_ports.append("%s.%s.%s" % (shelf, resource, port)) - else: - raise ValueError("Netmask and gateway must be specified") + # print("++++++++++++++++\n", self.ip_list, "++++++++++++++++\n") + for num_port in range(len(self.port_list)): + if self.ip_list[num_port] != 0: + if self.gateway is not None and self.netmask is not None: + shelf = self.local_realm.name_to_eid(self.port_list[num_port])[0] + resource = self.local_realm.name_to_eid(self.port_list[num_port])[1] + port = self.local_realm.name_to_eid(self.port_list[num_port])[2] + req_url = "/cli-json/set_port" + data = { + "shelf": shelf, + "resource": resource, + "port": port, + "ip_addr": self.ip_list[num_port], + "netmask": self.netmask, + "gateway": self.gateway + } + self.local_realm.json_post(req_url, data) + self.created_ports.append("%s.%s.%s" % (shelf, resource, port)) + else: + raise ValueError("Netmask and gateway must be specified") # if use test groups and test group does not exist, create cxs, create test group, assign to test group # if use test groups and test group exists and no cxs, create cxs, assign to test group @@ -310,7 +314,8 @@ class FileIOTest(LFCliBase): if self.wo_tg_exists: if not self.wo_tg_cx_exists: print("Creating Write Only CXs") - self.wo_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.wo_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) time.sleep(1) print("Adding cxs to %s" % self.wo_tg_profile.group_name) @@ -318,7 +323,8 @@ class FileIOTest(LFCliBase): self.wo_tg_profile.add_cx(cx) else: print("Creating Write Only CXs") - self.wo_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.wo_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) time.sleep(1) print("Creating Write Only test group") @@ -330,7 +336,8 @@ class FileIOTest(LFCliBase): if self.ro_tg_exists: if not self.ro_tg_cx_exists: print("Creating Read Only CXs") - self.ro_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.ro_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) time.sleep(1) print("Adding cxs to %s" % self.ro_tg_profile.group_name) @@ -338,7 +345,8 @@ class FileIOTest(LFCliBase): self.ro_tg_profile.add_cx(cx) else: print("Creating Read Only CXs") - self.ro_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.ro_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) time.sleep(1) print("Creating Read Only test group") @@ -350,7 +358,8 @@ class FileIOTest(LFCliBase): if self.wo_tg_exists: if not self.wo_tg_cx_exists: print("Creating Write Only CXs") - self.wo_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.wo_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) time.sleep(1) print("Adding cxs to %s" % self.wo_tg_profile.group_name) @@ -358,7 +367,8 @@ class FileIOTest(LFCliBase): self.wo_tg_profile.add_cx(cx) else: print("Creating Write Only CXs") - self.wo_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.wo_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) time.sleep(1) print("Creating Write Only test group") @@ -369,7 +379,8 @@ class FileIOTest(LFCliBase): if self.ro_tg_exists: if not self.ro_tg_cx_exists: print("Creating Read Only CXs") - self.ro_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.ro_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) time.sleep(1) print("Adding cxs to %s" % self.ro_tg_profile.group_name) @@ -377,7 +388,8 @@ class FileIOTest(LFCliBase): self.ro_tg_profile.add_cx(cx) else: print("Creating Read Only CXs") - self.ro_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.ro_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) time.sleep(1) print("Creating Read Only test group") @@ -390,21 +402,25 @@ class FileIOTest(LFCliBase): else: if self.mode == "write": print("Creating Write Only CXs") - self.wo_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.wo_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) elif self.mode == "read": print("Creating Read Only CXs") - self.ro_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.ro_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) elif self.mode == "both": print("Creating Write Only CXs") - self.wo_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.wo_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) print("Creating Read Only CXs") - self.ro_profile.create(ports=self.created_ports, sleep_time=.5, debug_=self.debug, + self.ro_profile.create(ports=self.created_ports, connections_per_port=self.connections_per_port, + sleep_time=.5, debug_=self.debug, suppress_related_commands_=None) else: - raise ValueError("Uknown mode used, must be (read, write, both)") + raise ValueError("Unknown mode used, must be (read, write, both)") else: raise ValueError("Mode must be set (read, write, both)") @@ -576,9 +592,14 @@ Generic command layout: default="AUTO") parser.add_argument('--server_mount', help='--server_mount The server to mount, ex: 192.168.100.5/exports/test1', default="10.40.0.1:/var/tmp/test") + parser.add_argument('--macvlan_parent', help='specifies parent port for macvlan creation', default=None) parser.add_argument('--first_port', help='specifies name of first port to be used', default=None) parser.add_argument('--num_ports', help='number of ports to create', default=1) + parser.add_argument('--connections_per_port', help='specifies number of connections to be used per port', default=1, + type=int) + parser.add_argument('--use_ports', help='list of comma separated ports to use with ips, \'=\' separates name and ip' + '{ port_name1=ip_addr1,port_name1=ip_addr2 }', default=None) parser.add_argument('--use_macvlans', help='will create macvlans', action='store_true', default=False) parser.add_argument('--first_mvlan_ip', help='specifies first static ip address to be used or dhcp', default=None) parser.add_argument('--netmask', help='specifies netmask to be used with static ip addresses', default=None) @@ -588,12 +609,11 @@ Generic command layout: parser.add_argument('--read_only_test_group', help='specifies name to use for read only test group', default=None) parser.add_argument('--write_only_test_group', help='specifies name to use for write only test group', default=None) parser.add_argument('--mode', help='write,read,both', default='both', type=str) - parser.add_argument('--use_ports', help='list of comma separated ports to use with ips, \'=\' separates name and ip' - '{ port_name1=ip_addr1,port_name1=ip_addr2 }', default=None) tg_group = parser.add_mutually_exclusive_group() tg_group.add_argument('--add_to_group', help='name of test group to add cxs to', default=None) tg_group.add_argument('--del_from_group', help='name of test group to delete cxs from', default=None) - parser.add_argument('--cxs', help='list of cxs to add/remove depending on use of --add_to_group or --del_from_group', default=None) + parser.add_argument('--cxs', help='list of cxs to add/remove depending on use of --add_to_group or --del_from_group' + , default=None) args = parser.parse_args() update_group_args = { @@ -647,7 +667,10 @@ Generic command layout: temp_list = args.use_ports.split(',') for port in temp_list: port_list.append(port.split('=')[0]) - ip_list.append(port.split('=')[1]) + if '=' in port: + ip_list.append(port.split('=')[1]) + else: + ip_list.append(0) if len(port_list) != len(ip_list): raise ValueError(temp_list, " ports must have matching ip addresses!") @@ -695,13 +718,13 @@ Generic command layout: write_only_test_group=args.write_only_test_group, read_only_test_group=args.read_only_test_group, update_group_args = update_group_args, + connections_per_port=args.connections_per_port, mode=args.mode # want a mount options param ) ip_test.cleanup(port_list) ip_test.build() - # exit(1) if not ip_test.passes(): print(ip_test.get_fail_message()) # ip_test.start(False, False) From d96df71c2a43b5447d692adf9b3412e3e6d916f6 Mon Sep 17 00:00:00 2001 From: Logan Lipke Date: Thu, 17 Dec 2020 16:48:50 -0800 Subject: [PATCH 11/35] Fixed unnecessary dabug=True in FIOEndpProfile --- py-json/realm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py-json/realm.py b/py-json/realm.py index 76a1cb8a..2c979a50 100755 --- a/py-json/realm.py +++ b/py-json/realm.py @@ -2487,7 +2487,7 @@ class FIOEndpProfile(LFCliBase): endp_data["directory"] = "/mnt/lf/wo_" + name + "_" + str(num_connection) + "_fio" url = "cli-json/add_file_endp" - self.local_realm.json_post(url, endp_data, debug_=True, suppress_related_commands_=suppress_related_commands_) + self.local_realm.json_post(url, endp_data, debug_=False, suppress_related_commands_=suppress_related_commands_) time.sleep(sleep_time) data = { From ea9fede527dc4d1793a1224a3cfc90eb6052cf19 Mon Sep 17 00:00:00 2001 From: Logan Lipke Date: Thu, 17 Dec 2020 16:49:08 -0800 Subject: [PATCH 12/35] Uncommented necesarry code --- py-scripts/test_fileio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py-scripts/test_fileio.py b/py-scripts/test_fileio.py index d969a086..439b0032 100755 --- a/py-scripts/test_fileio.py +++ b/py-scripts/test_fileio.py @@ -727,7 +727,7 @@ Generic command layout: ip_test.build() if not ip_test.passes(): print(ip_test.get_fail_message()) - # ip_test.start(False, False) + ip_test.start(False, False) ip_test.stop() if not ip_test.passes(): print(ip_test.get_fail_message()) From 1232cc6c673d43c44a9eac494545251e0d4d43ea Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Thu, 17 Dec 2020 08:42:38 -0800 Subject: [PATCH 13/35] check_large_files.sh: using mapfile, formatting --- check_large_files.sh | 49 ++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/check_large_files.sh b/check_large_files.sh index db9efae5..46d5692b 100755 --- a/check_large_files.sh +++ b/check_large_files.sh @@ -1,25 +1,29 @@ #!/bin/bash +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # +# Check for large files and purge many of the most inconsequencial # +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # set -x set -e +HR=" ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----" +funciton hr() { + echo "$HR" +} + eyedee=`id -u` if (( eyedee != 0 )); then - echo "Please become root to use this script" + echo "$0: Please become root to use this script, bye" exit 1 fi # Find core files core_files=() cd / -while read F; do - core_files+=("$F") -done < <(ls /core* /home/lanforge/core* 2>/dev/null) +mapfile -t core_files < <(ls /core* /home/lanforge/core* 2>/dev/null) # Find ath10k crash residue ath10_files=() -while read F; do - ath10_files+=("$F") -done < <(ls /home/lanforge/ath10* 2>/dev/null) +mapfile -t ath10_files < <(ls /home/lanforge/ath10* 2>/dev/null) # Find size of /mnt/lf that is not mounted cd /mnt @@ -31,18 +35,18 @@ usage_libmod=`du -sh *` # Find how many kernels are installed cd /boot -boot_kernels=(`ls init*`) +mapfile -t boot_kernels < <(ls init*) boot_usage=`du -sh .` -HR="---------------------------------------" -# # -# report sizes here # -# # + +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # +# report sizes here # +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # if (( ${#core_files[@]} > 0 )); then echo "Core Files:" - echo "$HR" - printf '%s\n' "${core_files[@]}" - echo "$HR" + hr + printf ' %s\n' "${core_files[@]}" + hr fi echo "Usage of /mnt: $usage_mnt" @@ -51,11 +55,16 @@ echo "Boot usage: $boot_usage" if (( ${#boot_kernels[@]} > 4 )); then echo "Boot ramdisks:" - echo "$HR" - printf '%s\n' "${boot_kernels[@]}" - echo "$HR" + hr + printf ' %s\n' "${boot_kernels[@]}" + hr fi -# delete extra things now # +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # +# delete extra things now # +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # + +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # +# ask to remove if we are interactive # +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # -# remove From 5cf4026dbf998a31977932bb098c9740b08c1fae Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Thu, 17 Dec 2020 08:44:24 -0800 Subject: [PATCH 14/35] renames to .bash --- check_large_files.sh => check_large_files.bash | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename check_large_files.sh => check_large_files.bash (100%) diff --git a/check_large_files.sh b/check_large_files.bash similarity index 100% rename from check_large_files.sh rename to check_large_files.bash From b3d88f672d585748bad7fd0693945560a3cbdea2 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Thu, 17 Dec 2020 08:45:23 -0800 Subject: [PATCH 15/35] typo --- check_large_files.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_large_files.bash b/check_large_files.bash index 46d5692b..e1e88f5e 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -5,7 +5,7 @@ set -x set -e HR=" ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----" -funciton hr() { +function hr() { echo "$HR" } From b43f6edb6d8d0363562e65346d93675879267cbb Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Thu, 17 Dec 2020 09:55:14 -0800 Subject: [PATCH 16/35] check_large_files.bash: WIP --- check_large_files.bash | 116 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 10 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index e1e88f5e..f339144d 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -2,12 +2,24 @@ # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # Check for large files and purge many of the most inconsequencial # # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # -set -x -set -e -HR=" ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----" -function hr() { - echo "$HR" -} +# set -x +# set -e + +USAGE="$0 # Check for large files and purge many of the most inconsequencial + -a # automatic: disable menu and clean automatically + -b # remove extra kernels and modules + -c # remove all core files + -d # remove old LANforge downloads + -h # help + -k # remove ath10k crash files + -l # remove old files from /var/log, truncate /var/log/messages + -m # remove orphaned fileio items in /mnt/lf + -q # quiet + -r # compress .csv data in /home/lanforge + -t # remove /var/tmp files + -v # verbose + +" eyedee=`id -u` if (( eyedee != 0 )); then @@ -15,12 +27,83 @@ if (( eyedee != 0 )); then exit 1 fi +# these are default selections +selections=() +deletion_targets=() +show_menu=1 +verbose=0 +quiet=0 +#opts="" +opts="abcdhklmqrtv" +while getopts $opts opt; do + case "$opt" in + a) + verbose=0 + quiet=1 + selections+=($opt) + show_menu=0 + ;; + b) + selections+=($opt) + ;; + c) + selections+=($opt) + ;; + d) + selections+=($opt) + ;; + h) + echo "$USAGE" + exit 0 + ;; + k) + selections+=($opt) + ;; + l) + selections+=($opt) + ;; + m) + selections+=($opt) + ;; + r) + selections+=($opt) + ;; + q) + quiet=1 + verbose=0 + selections+=($opt) + ;; + t) + selections+=($opt) + ;; + v) + quiet=0 + verbose=1 + selections+=($opt) + ;; + *) + echo "unknown option: $opt" + echo "$USAGE" + exit 1 + ;; + esac +done + +if (( ${#selections} < 1 )); then + echo "$USAGE" + exit 0 +fi + +HR=" ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----" +function hr() { + echo "$HR" +} + # Find core files core_files=() cd / mapfile -t core_files < <(ls /core* /home/lanforge/core* 2>/dev/null) - # Find ath10k crash residue ath10_files=() mapfile -t ath10_files < <(ls /home/lanforge/ath10* 2>/dev/null) @@ -53,7 +136,7 @@ echo "Usage of /mnt: $usage_mnt" echo "Usage of /lib/modules: $usage_libmod" echo "Boot usage: $boot_usage" -if (( ${#boot_kernels[@]} > 4 )); then +if (( ${#boot_kernels[@]} > 1 )); then echo "Boot ramdisks:" hr printf ' %s\n' "${boot_kernels[@]}" @@ -63,8 +146,21 @@ fi # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # delete extra things now # # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # - +echo "Automatic deletion will include: " +echo " journalctl space" +sleep 1 # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # ask to remove if we are interactive # # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # - +if +item="" +while [[ $item != q ]]; do + hr + echo "Would you like to delete? " + echo " 1) core crash files" + echo " 2) ath10k crash files" + echo " 3) old var/www downloads" + echo " 4) old lanforge downloads" + echo " 5) orphaned /mnt/lf files" + read -p "[1-5] or q ? " item +done From 68a30d6bd21c9c698623473fb0ef9488bf7e0252 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Thu, 17 Dec 2020 20:48:31 -0800 Subject: [PATCH 17/35] lf_icemod.pl: fixes connection logic --- lf_icemod.pl | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/lf_icemod.pl b/lf_icemod.pl index f6ec3d5a..37ffeed3 100755 --- a/lf_icemod.pl +++ b/lf_icemod.pl @@ -163,19 +163,10 @@ $t->waitfor("/btbits\>\>/"); my $dt = ""; my $utils = new LANforge::Utils(); -$utils->telnet($t); -$utils->cli_send_silent(0); # Show input to CLI -if ($quiet & 0x1) { - $utils->cli_rcv_silent(1); -} -else { - $utils->cli_rcv_silent(0); -} +$utils->connect($lfmgr_host, $lfmgr_port); -# $utils->doCmd("log_level 63"); my $cmd; - $speed = "NA" if ($speed eq ""); $latency = "NA" if ($latency eq ""); $max_jitter = "NA" if ($max_jitter eq ""); From d3fab7c7ff089b2d18ae7508128404dbbcffa2fa Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Thu, 17 Dec 2020 20:49:34 -0800 Subject: [PATCH 18/35] check_large_files.bash: adds mapfile --- check_large_files.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_large_files.bash b/check_large_files.bash index f339144d..e23ffa81 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -114,7 +114,7 @@ usage_mnt=`du -shxc .` # Find size of /lib/modules cd /lib/modules -usage_libmod=`du -sh *` +mapfile -t usage_libmod < <(du -sh *) # Find how many kernels are installed cd /boot From 062cf04d9be646fca6aa357df37102478a08358d Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 12:36:26 -0800 Subject: [PATCH 19/35] lfcli_base.py: adds proxy construction logic --- py-json/LANforge/LFRequest.py | 145 +++++++++++++++++++++------------ py-json/LANforge/lfcli_base.py | 115 +++++++++++++++++++------- 2 files changed, 179 insertions(+), 81 deletions(-) diff --git a/py-json/LANforge/LFRequest.py b/py-json/LANforge/LFRequest.py index 99761ea5..3baed36c 100644 --- a/py-json/LANforge/LFRequest.py +++ b/py-json/LANforge/LFRequest.py @@ -8,9 +8,12 @@ if sys.version_info[0] != 3: exit() import pprint -import urllib.request -import urllib.error -import urllib.parse +import urllib +import time +from urllib import request +from urllib import error +from urllib import parse + import json from LANforge import LFUtils @@ -33,11 +36,27 @@ class LFRequest: # please see this discussion on ProxyHandlers: # https://docs.python.org/3/library/urllib.request.html#urllib.request.ProxyHandler - if proxies_ is not None: - # check to see if proxy has some fields - if ("host" not in proxies_) or ("user" not in proxies_): - raise ValueError("HTTP proxy requires, host, user, pass values.") - self.proxies = proxies_; + # but this makes much more sense: + # https://gist.github.com/aleiphoenix/4159510 + + # if debug_: + # if proxies_ is None: + # print("LFRequest_init_: no proxies_") + # else: + # print("LFRequest: proxies_: ") + # pprint.pprint(proxies_) + + if (proxies_ is not None) and (len(proxies_) > 0): + if ("http" not in proxies_) and ("https" not in proxies_): + raise ValueError("Neither http or https set in proxy definitions. Expects proxy={'http':, 'https':, }") + self.proxies = proxies_ + + # if debug_: + # if self.proxies is None: + # print("LFRequest_init_: no proxies") + # else: + # print("LFRequest: proxies: ") + # pprint.pprint(self.proxies) if not url.startswith("http://") and not url.startswith("https://"): print("No http:// or https:// found, prepending http:// to "+url) @@ -50,7 +69,7 @@ class LFRequest: self.requested_url = url if self.requested_url is None: - raise Exception("Bad LFRequest of url[%s] uri[%s] -> None" % url, uri) + raise Exception("Bad LFRequest of url[%s] uri[%s] -> None" % (url, uri)) if self.requested_url.find('//'): protopos = self.requested_url.find("://") @@ -65,14 +84,6 @@ class LFRequest: if self.debug: print("new LFRequest[%s]" % self.requested_url ) - def update_proxies(self, request): - if (request is None) or (self.proxies is None): - return - - for (proto, host) in self.proxies.items(): - request.set_proxy(host, proto) - - # request first url on stack def formPost(self, show_error=True, debug=False, die_on_error_=False): return self.form_post(show_error=show_error, debug=debug, die_on_error_=die_on_error_) @@ -84,6 +95,13 @@ class LFRequest: debug = True responses = [] urlenc_data = "" + # https://stackoverflow.com/a/59635684/11014343 + if (self.proxies is not None) and (len(self.proxies) > 0): + # https://stackoverflow.com/a/59635684/11014343 + opener = request.build_opener(request.ProxyHandler(self.proxies)) + request.install_opener(opener) + + if (debug): print("formPost: url: "+self.requested_url) if ((self.post_data != None) and (self.post_data is not self.No_Data)): @@ -91,32 +109,31 @@ class LFRequest: if (debug): print("formPost: data looks like:" + str(urlenc_data)) print("formPost: url: "+self.requested_url) - request = urllib.request.Request(url=self.requested_url, - data=urlenc_data, - headers=self.default_headers) + myrequest = request.Request(url=self.requested_url, + data=urlenc_data, + headers=self.default_headers) else: - request = urllib.request.Request(url=self.requested_url, headers=self.default_headers) + myrequest = request.Request(url=self.requested_url, headers=self.default_headers) print("No data for this formPost?") - self.update_proxies(request) + myrequest.headers['Content-type'] = 'application/x-www-form-urlencoded' - request.headers['Content-type'] = 'application/x-www-form-urlencoded' resp = '' try: - resp = urllib.request.urlopen(request) + resp = urllib.request.urlopen(myrequest) responses.append(resp) return responses[0] except urllib.error.HTTPError as error: if (show_error): print("----- LFRequest::formPost:76 HTTPError: --------------------------------------------") - print("%s: %s; URL: %s"%(error.code, error.reason, request.get_full_url())) + print("%s: %s; URL: %s"%(error.code, error.reason, myrequest.get_full_url())) LFUtils.debug_printer.pprint(error.headers) #print("Error: ", sys.exc_info()[0]) #print("Request URL:", request.get_full_url()) - print("Request Content-type:", request.get_header('Content-type')) - print("Request Accept:", request.get_header('Accept')) + print("Request Content-type:", myrequest.get_header('Content-type')) + print("Request Accept:", myrequest.get_header('Accept')) print("Request Data:") - LFUtils.debug_printer.pprint(request.data) + LFUtils.debug_printer.pprint(myrequest.data) if (len(responses) > 0): print("----- Response: --------------------------------------------------------") LFUtils.debug_printer.pprint(responses[0].reason) @@ -127,7 +144,7 @@ class LFRequest: except urllib.error.URLError as uerror: if show_error: print("----- LFRequest::formPost:94 URLError: ---------------------------------------------") - print("Reason: %s; URL: %s"%(uerror.reason, request.get_full_url())) + print("Reason: %s; URL: %s"%(uerror.reason, myrequest.get_full_url())) print("------------------------------------------------------------------------") if (die_on_error_ == True) or (self.die_on_error == True): exit(1) @@ -142,18 +159,25 @@ class LFRequest: if self.die_on_error: die_on_error_ = True responses = [] + if (self.proxies is not None) and (len(self.proxies) > 0): + opener = request.build_opener(request.ProxyHandler(self.proxies)) + request.install_opener(opener) + if ((self.post_data != None) and (self.post_data is not self.No_Data)): - request = urllib.request.Request(url=self.requested_url, - method=method_, - data=json.dumps(self.post_data).encode("utf-8"), - headers=self.default_headers) + myrequest = request.Request(url=self.requested_url, + method=method_, + data=json.dumps(self.post_data).encode("utf-8"), + headers=self.default_headers) else: - request = urllib.request.Request(url=self.requested_url, headers=self.default_headers) + myrequest = request.Request(url=self.requested_url, headers=self.default_headers) print("No data for this jsonPost?") - request.headers['Content-type'] = 'application/json' + myrequest.headers['Content-type'] = 'application/json' + + # https://stackoverflow.com/a/59635684/11014343 + try: - resp = urllib.request.urlopen(request) + resp = request.urlopen(myrequest) resp_data = resp.read().decode('utf-8') if (debug): print("----- LFRequest::json_post:128 debug: --------------------------------------------") @@ -176,14 +200,14 @@ class LFRequest: except urllib.error.HTTPError as error: if show_error or die_on_error_ or (error.code != 404): print("----- LFRequest::json_post:147 HTTPError: --------------------------------------------") - print("<%s> HTTP %s: %s" % (request.get_full_url(), error.code, error.reason )) + print("<%s> HTTP %s: %s" % (myrequest.get_full_url(), error.code, error.reason )) print("Error: ", sys.exc_info()[0]) - print("Request URL:", request.get_full_url()) - print("Request Content-type:", request.get_header('Content-type')) - print("Request Accept:", request.get_header('Accept')) + print("Request URL:", myrequest.get_full_url()) + print("Request Content-type:", myrequest.get_header('Content-type')) + print("Request Accept:", myrequest.get_header('Accept')) print("Request Data:") - LFUtils.debug_printer.pprint(request.data) + LFUtils.debug_printer.pprint(myrequest.data) if error.headers: # the HTTPError is of type HTTPMessage a subclass of email.message @@ -200,7 +224,7 @@ class LFRequest: except urllib.error.URLError as uerror: if show_error: print("----- LFRequest::json_post:171 URLError: ---------------------------------------------") - print("Reason: %s; URL: %s"%(uerror.reason, request.get_full_url())) + print("Reason: %s; URL: %s"%(uerror.reason, myrequest.get_full_url())) print("------------------------------------------------------------------------") if (die_on_error_ == True) or (self.die_on_error == True): exit(1) @@ -225,13 +249,21 @@ class LFRequest: die_on_error_ = True if debug: print("LFUtils.get: url: "+self.requested_url) - myrequest = urllib.request.Request(url=self.requested_url, - headers=self.default_headers, - method=method_) - self.update_proxies(myrequest) + + # https://stackoverflow.com/a/59635684/11014343 + if (self.proxies is not None) and (len(self.proxies) > 0): + opener = request.build_opener(request.ProxyHandler(self.proxies)) + #opener = urllib.request.build_opener(myrequest.ProxyHandler(self.proxies)) + request.install_opener(opener) + + myrequest = request.Request(url=self.requested_url, + headers=self.default_headers, + method=method_) + + myresponses = [] try: - myresponses.append(urllib.request.urlopen(myrequest)) + myresponses.append(request.urlopen(myrequest)) return myresponses[0] except urllib.error.HTTPError as error: if debug: @@ -290,12 +322,25 @@ class LFRequest: self.post_data = data -def plain_get(url_=None, debug_=False, die_on_error_=False): - myrequest = urllib.request.Request(url=url_) +def plain_get(url_=None, debug_=False, die_on_error_=False, proxies_=None): + """ + This static method does not respect LFRequest.proxy, it is not set in scope here + :param url_: + :param debug_: + :param die_on_error_: + :return: + """ + myrequest = request.Request(url=url_) myresponses = [] try: - myresponses.append(urllib.request.urlopen(myrequest)) + if (proxies_ is not None) and (len(proxies_) > 0): + # https://stackoverflow.com/a/59635684/11014343 + opener = myrequest.build_opener(myrequest.ProxyHandler(proxies_)) + myrequest.install_opener(opener) + + myresponses.append(request.urlopen(myrequest)) return myresponses[0] + except urllib.error.HTTPError as error: if debug_: print("----- LFRequest::get:181 HTTPError: --------------------------------------------") diff --git a/py-json/LANforge/lfcli_base.py b/py-json/LANforge/lfcli_base.py index 1bac0034..ed8901ba 100644 --- a/py-json/LANforge/lfcli_base.py +++ b/py-json/LANforge/lfcli_base.py @@ -27,15 +27,23 @@ class LFCliBase: _exit_on_error=False, _exit_on_fail=False, _local_realm=None, + _proxy_str=None, _capture_signal_list=[]): self.fail_pref = "FAILED: " self.pass_pref = "PASSED: " self.lfclient_host = _lfjson_host self.lfclient_port = _lfjson_port self.debug = _debug + # if (_debug): + # print("LFCliBase._proxy_str: %s" % _proxy_str) + self.proxy = {} + self.adjust_proxy(_proxy_str) + if (_local_realm is not None): self.local_realm = _local_realm + # if (_debug): + # print("LFCliBase._proxy_str: %s" % _proxy_str) self.lfclient_url = "http://%s:%s" % (self.lfclient_host, self.lfclient_port) self.test_results = [] self.halt_on_error = _halt_on_error @@ -144,7 +152,6 @@ class LFCliBase: else: if self.debug: print("subclass ignored signal") - def clear_test_results(self): self.test_results.clear() @@ -160,8 +167,13 @@ class LFCliBase: :return: http response object """ json_response = None + debug_ |= self.debug try: - lf_r = LFRequest.LFRequest(self.lfclient_url, _req_url, debug_=self.debug, die_on_error_=self.exit_on_error) + lf_r = LFRequest.LFRequest(url=self.lfclient_url, + uri=_req_url, + proxies_=self.proxy, + debug_=debug_, + die_on_error_=self.exit_on_error) if suppress_related_commands_ is None: if 'suppress_preexec_cli' in _data: del _data['suppress_preexec_cli'] @@ -183,16 +195,16 @@ class LFCliBase: _data['suppress_postexec_method'] = True lf_r.addPostData(_data) - if debug_ or self.debug: + if debug_: LANforge.LFUtils.debug_printer.pprint(_data) - json_response = lf_r.jsonPost(show_error=self.debug, - debug=(self.debug or debug_), + json_response = lf_r.json_post(show_error=debug_, + debug=debug_, response_json_list_=response_json_list_, die_on_error_=self.exit_on_error) if debug_ and (response_json_list_ is not None): pprint.pprint(response_json_list_) except Exception as x: - if self.debug or self.halt_on_error or self.exit_on_error: + if debug_ or self.halt_on_error or self.exit_on_error: print("json_post posted to %s" % _req_url) pprint.pprint(_data) print("Exception %s:" % x) @@ -212,20 +224,25 @@ class LFCliBase: :param response_json_list_: array for json results in the response object, (alternative return method) :return: http response object """ + debug_ |= self.debug json_response = None try: - lf_r = LFRequest.LFRequest(self.lfclient_url, _req_url, debug_=self.debug, die_on_error_=self.exit_on_error) + lf_r = LFRequest.LFRequest(url=self.lfclient_url, + uri=_req_url, + proxies_=self.proxy, + debug_=debug_, + die_on_error_=self.exit_on_error) lf_r.addPostData(_data) - if debug_ or self.debug: + if debug_: LANforge.LFUtils.debug_printer.pprint(_data) json_response = lf_r.json_put(show_error=self.debug, - debug=(self.debug or debug_), + debug=debug_, response_json_list_=response_json_list_, die_on_error_=self.exit_on_error) if debug_ and (response_json_list_ is not None): pprint.pprint(response_json_list_) except Exception as x: - if self.debug or self.halt_on_error or self.exit_on_error: + if debug_ or self.halt_on_error or self.exit_on_error: print("json_put submitted to %s" % _req_url) pprint.pprint(_data) print("Exception %s:" % x) @@ -235,19 +252,26 @@ class LFCliBase: return json_response def json_get(self, _req_url, debug_=False): - if self.debug or debug_: - print("GET: "+_req_url) + debug_ |= self.debug + # if debug_: + # print("json_get: "+_req_url) + # print("json_get: proxies:") + # pprint.pprint(self.proxy) json_response = None # print("----- GET ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ") try: - lf_r = LFRequest.LFRequest(self.lfclient_url, _req_url, debug_=(self.debug or debug_), die_on_error_=self.exit_on_error) - json_response = lf_r.get_as_json(debug_=self.debug, die_on_error_=self.halt_on_error) + lf_r = LFRequest.LFRequest(url=self.lfclient_url, + uri=_req_url, + proxies_=self.proxy, + debug_=debug_, + die_on_error_=self.exit_on_error) + json_response = lf_r.get_as_json(debug_=debug_, die_on_error_=self.halt_on_error) #debug_printer.pprint(json_response) - if (json_response is None) and (self.debug or debug_): + if (json_response is None) and debug_: print("LFCliBase.json_get: no entity/response, probabily status 404") return None except ValueError as ve: - if self.debug or self.halt_on_error or self.exit_on_error: + if debug_ or self.halt_on_error or self.exit_on_error: print("jsonGet asked for " + _req_url) print("Exception %s:" % ve) traceback.print_exception(ValueError, ve, ve.__traceback__, chain=True) @@ -257,22 +281,24 @@ class LFCliBase: return json_response def json_delete(self, _req_url, debug_=False): - if self.debug or debug_: + debug_ |= self.debug + if debug_: print("DELETE: "+_req_url) json_response = None try: # print("----- DELETE ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ") - lf_r = LFRequest.LFRequest(self.lfclient_url, _req_url, - debug_=(self.debug or debug_), + lf_r = LFRequest.LFRequest(url=self.lfclient_url, + uri=_req_url, + proxies_=self.proxy, + debug_=debug_, die_on_error_=self.exit_on_error) - json_response = lf_r.json_delete(debug=self.debug, - die_on_error_=self.halt_on_error) + json_response = lf_r.json_delete(debug=debug_, die_on_error_=self.halt_on_error) #debug_printer.pprint(json_response) - if (json_response is None) and (self.debug or debug_): + if (json_response is None) and debug_: print("LFCliBase.json_delete: no entity/response, probabily status 404") return None except ValueError as ve: - if self.debug or self.halt_on_error or self.exit_on_error: + if debug_ or self.halt_on_error or self.exit_on_error: print("json_delete asked for " + _req_url) print("Exception %s:" % ve) traceback.print_exception(ValueError, ve, ve.__traceback__, chain=True) @@ -356,17 +382,14 @@ class LFCliBase: pass_list.append(result) return pass_list - def get_pass_message(self): pass_messages = self.get_passed_result_list() return "\n".join(pass_messages) - def get_fail_message(self): fail_messages = self.get_failed_result_list() return "\n".join(fail_messages) - def get_all_message(self): return "\n".join(self.test_results) @@ -390,7 +413,6 @@ class LFCliBase: print(message %(fail_len,total_len)) sys.exit(1) - # use this inside the class to log a failure result and print it if wished def _fail(self, message, print_=False): self.test_results.append(self.fail_pref + message) @@ -406,13 +428,38 @@ class LFCliBase: print(message %(num_passing,num_total)) sys.exit(0) - # use this inside the class to log a pass result and print if wished. def _pass(self, message, print_=False): self.test_results.append(self.pass_pref + message) if print_: print(self.pass_pref + message) - #Create argparse with radio, securiy, ssid and passwd required + + def adjust_proxy(self, proxy_str): + # if self.debug: + # print("lfclibase.adjust_proxy: %s" % proxy_str) + if (proxy_str is None) or (proxy_str == ""): + return + if self.proxy is None: + self.proxy = {} + + if proxy_str.find("http:") > -1: + self.proxy["http"] = proxy_str + if proxy_str.find("https:") > -1: + self.proxy["https"] = proxy_str + # if self.debug: + # print("lfclibase::self.proxy: ") + # pprint.pprint(self.proxy) + + + # This style of Action subclass for argparse can't do much unless we incorporate + # our argparse as a member of LFCliBase. Then we can do something like automatically + # parse our proxy string without using _init_ arguments + # class ProxyAction(argparse.Action, zelf): + # def __init__(self, outter_): + # pass + # def __call__(self, parser, namespace, values, option_string=None): + # zelf.adjust_proxy(values) + @staticmethod def create_bare_argparse(prog=None, formatter_class=None, epilog=None, description=None): if (prog is not None) or (formatter_class is not None) or (epilog is not None) or (description is not None): @@ -428,13 +475,18 @@ class LFCliBase: optional.add_argument('--mgr', help='hostname for where LANforge GUI is running', default='localhost') optional.add_argument('--mgr_port', help='port LANforge GUI HTTP service is running on', default=8080) optional.add_argument('--debug', help='Enable debugging', default=False, action="store_true") + optional.add_argument('--proxy', nargs='?', default=None, # action=ProxyAction, + help='Connection proxy like http://proxy.localnet:80 or https://user:pass@proxy.localnet:3128') return parser # Create argparse with radio, securiy, ssid and passwd required # TODO: show example of how to add required or optional arguments from calling class @staticmethod - def create_basic_argparse(prog=None, formatter_class=None, epilog=None, description=None): + def create_basic_argparse(prog=None, + formatter_class=None, + epilog=None, + description=None): if (prog is not None) or (formatter_class is not None) or (epilog is not None) or (description is not None): parser = argparse.ArgumentParser(prog=prog, formatter_class=formatter_class, @@ -442,7 +494,6 @@ class LFCliBase: description=description) else: parser = argparse.ArgumentParser() - optional = parser.add_argument_group('optional arguments') required = parser.add_argument_group('required arguments') #Optional Args @@ -454,6 +505,8 @@ class LFCliBase: optional.add_argument('--num_stations', help='Number of stations to create', default=0) optional.add_argument('--test_id', help='Test ID (intended to use for ws events)', default="webconsole") optional.add_argument('--debug', help='Enable debugging', default=False, action="store_true") + optional.add_argument('--proxy', nargs='?', default=None, + help='Connection proxy like http://proxy.localnet:80 or https://user:pass@proxy.localnet:3128') #Required Args required.add_argument('--radio', help='radio EID, e.g: 1.wiphy2', required=True) required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True) From c1bd64adb9ed3eed52004ee7c5a9f8165e5868f2 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 12:36:41 -0800 Subject: [PATCH 20/35] uses non-deprecated method --- py-json/LANforge/LFUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py-json/LANforge/LFUtils.py b/py-json/LANforge/LFUtils.py index 5ea2d86b..62766311 100644 --- a/py-json/LANforge/LFUtils.py +++ b/py-json/LANforge/LFUtils.py @@ -459,7 +459,7 @@ def wait_until_ports_disappear(base_url="http://localhost:8080", port_list=[], d if debug: print("checking:" + check_url) lf_r = LFRequest.LFRequest(base_url, check_url) - json_response = lf_r.getAsJson(debug_=debug) + json_response = lf_r.get_as_json(debug_=debug) if (json_response != None): found_stations.append(port_name) if len(found_stations) > 0: From 2a8b0e14335cb605a9c9f622615798f9967bc11b Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 12:38:10 -0800 Subject: [PATCH 21/35] realm.py: adds appropriate parameters to init and uses super init with capture_signals and proxy --- py-json/realm.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/py-json/realm.py b/py-json/realm.py index 2c979a50..2d34568f 100755 --- a/py-json/realm.py +++ b/py-json/realm.py @@ -44,10 +44,39 @@ def wpa_ent_list(): ] class Realm(LFCliBase): - def __init__(self, lfclient_host="localhost", lfclient_port=8080, debug_=False, halt_on_error_=False): - super().__init__(_lfjson_host=lfclient_host, _lfjson_port=lfclient_port, _debug=debug_, _halt_on_error=halt_on_error_) + def __init__(self, + lfclient_host="localhost", + lfclient_port=8080, + debug_=False, + halt_on_error_=False, + _exit_on_error=False, + _exit_on_fail=False, + _local_realm=None, + _proxy_str=None, + _capture_signal_list=[]): + super().__init__(_lfjson_host=lfclient_host, + _lfjson_port=lfclient_port, + _debug=debug_, + _halt_on_error=halt_on_error_, + _exit_on_error=_exit_on_error, + _exit_on_fail=_exit_on_fail, + _proxy_str=_proxy_str, + _capture_signal_list=_capture_signal_list) # self.lfclient_url = "http://%s:%s" % (lfclient_host, lfclient_port) + super().__init__(lfclient_host, + lfclient_port, + _debug=debug_, + _halt_on_error=halt_on_error_, + _exit_on_error=_exit_on_error, + _exit_on_fail=_exit_on_fail, + #_local_realm=self, + _proxy_str=_proxy_str, + _capture_signal_list=_capture_signal_list) + self.debug = debug_ + # if debug_: + # print("Realm _proxy_str: %s" % _proxy_str) + # pprint(_proxy_str) self.check_connect() self.chan_to_freq = {} self.freq_to_chan = {} From 0673825f9e8850f1cba8e666570ff3b1f5cb126f Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 12:39:16 -0800 Subject: [PATCH 22/35] test_ipv4_connection.py: updates init and uses proxy, uses localrealm constructor in super constructor call --- py-scripts/test_ipv4_connection.py | 72 ++++++++++++++++++------------ 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/py-scripts/test_ipv4_connection.py b/py-scripts/test_ipv4_connection.py index f2793f28..713c238e 100755 --- a/py-scripts/test_ipv4_connection.py +++ b/py-scripts/test_ipv4_connection.py @@ -20,35 +20,42 @@ import pprint class IPv4Test(LFCliBase): - def __init__(self, - ssid, - security, - password, - host, - port, - sta_list=None, - number_template="00000", - radio="wiphy0", + def __init__(self, + _ssid=None, + _security=None, + _password=None, + _host=None, + _port=None, + _sta_list=None, + _number_template="00000", + _radio="wiphy0", + _proxy_str=None, _debug_on=False, _exit_on_error=False, _exit_on_fail=False): - super().__init__(host, - port, + super().__init__(_host, + _port, + _proxy_str=_proxy_str, + _local_realm=realm.Realm(lfclient_host=_host, + lfclient_port=_port, + halt_on_error_=_exit_on_error, + _exit_on_error=_exit_on_error, + _exit_on_fail=_exit_on_fail, + _proxy_str=_proxy_str, + debug_=_debug_on), _debug=_debug_on, _halt_on_error=_exit_on_error, _exit_on_fail=_exit_on_fail) - self.host = host - self.port = port - self.ssid = ssid - self.security = security - self.password = password - self.sta_list = sta_list - self.radio = radio + self.host = _host + self.port = _port + self.ssid = _ssid + self.security = _security + self.password = _password + self.sta_list = _sta_list + self.radio = _radio self.timeout = 120 - self.number_template = number_template + self.number_template = _number_template self.debug = _debug_on - self.local_realm = realm.Realm(lfclient_host=self.host, lfclient_port=self.port) - self.station_profile = self.local_realm.new_station_profile() self.station_profile.lfclient_url = self.lfclient_url self.station_profile.ssid = self.ssid @@ -137,7 +144,7 @@ def main(): description='''\ test_ipv4_connection.py -------------------- -Generic command example: +Command example: ./test_ipv4_connection.py --upstream_port eth1 --radio wiphy0 @@ -151,6 +158,9 @@ Generic command example: required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True) args = parser.parse_args() + #if args.debug: + # pprint.pprint(args) + # time.sleep(5) if (args.radio is None): raise ValueError("--radio required") @@ -164,14 +174,18 @@ Generic command example: end_id=num_sta-1, padding_number=10000, radio=args.radio) - - ip_test = IPv4Test(host=args.mgr, port=args.mgr_port, - ssid=args.ssid, - password=args.passwd, - security=args.security, - sta_list=station_list, - radio=args.radio, + if args.debug: + print("args.proxy: %s" % args.proxy) + ip_test = IPv4Test(_host=args.mgr, + _port=args.mgr_port, + _ssid=args.ssid, + _password=args.passwd, + _security=args.security, + _sta_list=station_list, + _radio=args.radio, + _proxy_str=args.proxy, _debug_on=args.debug) + ip_test.cleanup(station_list) ip_test.build() if not ip_test.passes(): From 5e767b0bb37e868d48eafd73ea4e23ff0ffc97c6 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 11:09:52 -0800 Subject: [PATCH 23/35] test_all_scripts.sh: reformatted --- py-scripts/test_all_scripts.sh | 125 ++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 58 deletions(-) diff --git a/py-scripts/test_all_scripts.sh b/py-scripts/test_all_scripts.sh index 3fc68a92..1d277311 100755 --- a/py-scripts/test_all_scripts.sh +++ b/py-scripts/test_all_scripts.sh @@ -1,7 +1,7 @@ #!/bin/bash #This bash script aims to automate the test process of all Candela Technologies's test_* scripts in the lanforge-scripts directory. The script can be run 2 ways and may include (via user input) the "start_num" and "stop_num" variables to select which tests should be run. # OPTION ONE: ./test_all_scripts.sh : this command runs all the scripts in the array "testCommands" -# OPTION TWO: ./test_all_scripts.sh 4 5 : this command runs py-script commands (in testCommands array) that include the py-script options beginning with 4 and 5 (inclusive) in case function ret_case_num. +# OPTION TWO: ./test_all_scripts.sh 4 5 : this command runs py-script commands (in testCommands array) that include the py-script options beginning with 4 and 5 (inclusive) in case function ret_case_num. #Variables NUM_STA=4 SSID_USED="jedway-wpa2-x2048-5-3" @@ -13,78 +13,87 @@ CURR_TEST_NUM=0 STOP_NUM=9 START_NUM=0 #Test array -testCommands=("./example_wpa_connection.py --num_stations $NUM_STA --ssid jedway-r8000-36 --passwd jedway-r8000-36 --radio $RADIO_USED --security wpa" -"./example_wpa2_connection.py --num_stations $NUM_STA --ssid $SSID_USED --passwd $SSID_USED --radio $RADIO_USED --security wpa2" -"./example_wep_connection.py --num_stations $NUM_STA --ssid jedway-wep-48 --passwd jedway-wep-48 --radio $RADIO_USED --security wep" -"./example_wpa3_connection.py --num_stations $NUM_STA --ssid jedway-wpa3-1 --passwd jedway-wpa3-1 --radio $RADIO_USED --security wpa3" -"./test_ipv4_connection.py --radio wiphy2 --num_stations $NUM_STA --ssid $SSID_USED --passwd $PASSWD_USED --security $SECURITY --debug --upstream_port eth1" -"./test_generic.py --mgr localhost --mgr_port 4122 --radio $RADIO_USED --ssid SSID_USED --passwd $PASSWD_USED --num_stations $NUM_STA --type lfping --dest 10.40.0.1 --security $SECURITY" -"./test_generic.py --mgr localhost --mgr_port 4122 --radio $RADIO_USED --ssid $SSID_USED --passwd $PASSWD_USED --num_stations $NUM_STA --type speedtest --speedtest_min_up 20 --speedtest_min_dl 20 --speedtest_max_ping 150 --security $SECURITY" -"./test_ipv4_l4_urls_per_ten.py --upstream_port eth1 --radio $RADIO_USED --num_stations $NUM_STA --security $SECURITY --ssid $SSID_USED --passwd $PASSWD_USED --num_tests 1 --requests_per_ten 600 --target_per_ten 600" -"./test_ipv4_l4_wifi.py --upstream_port eth1 --radio wiphy0 --num_stations $NUM_STA --security $SECURITY --ssid jedway-wpa2-x2048-4-4 --passwd jedway-wpa2-x2048-4-4 --test_duration 3m" -"./test_ipv4_l4.py --radio wiphy3 --num_stations 4 --security wpa2 --ssid jedway-wpa2-x2048-4-1 --passwd jedway-wpa2-x2048-4-1 --url \"dl http://10.40.0.1 /dev/null\" --test_duration 2m --debug" +testCommands=("./example_wpa_connection.py --num_stations $NUM_STA --ssid jedway-r8000-36 --passwd jedway-r8000-36 --radio $RADIO_USED --security wpa" + "./example_wpa2_connection.py --num_stations $NUM_STA --ssid $SSID_USED --passwd $SSID_USED --radio $RADIO_USED --security wpa2" + "./example_wep_connection.py --num_stations $NUM_STA --ssid jedway-wep-48 --passwd jedway-wep-48 --radio $RADIO_USED --security wep" + "./example_wpa3_connection.py --num_stations $NUM_STA --ssid jedway-wpa3-1 --passwd jedway-wpa3-1 --radio $RADIO_USED --security wpa3" + "./test_ipv4_connection.py --radio wiphy2 --num_stations $NUM_STA --ssid $SSID_USED --passwd $PASSWD_USED --security $SECURITY --debug --upstream_port eth1" + "./test_generic.py --mgr localhost --mgr_port 4122 --radio $RADIO_USED --ssid SSID_USED --passwd $PASSWD_USED --num_stations $NUM_STA --type lfping --dest 10.40.0.1 --security $SECURITY" + "./test_generic.py --mgr localhost --mgr_port 4122 --radio $RADIO_USED --ssid $SSID_USED --passwd $PASSWD_USED --num_stations $NUM_STA --type speedtest --speedtest_min_up 20 --speedtest_min_dl 20 --speedtest_max_ping 150 --security $SECURITY" + "./test_ipv4_l4_urls_per_ten.py --upstream_port eth1 --radio $RADIO_USED --num_stations $NUM_STA --security $SECURITY --ssid $SSID_USED --passwd $PASSWD_USED --num_tests 1 --requests_per_ten 600 --target_per_ten 600" + "./test_ipv4_l4_wifi.py --upstream_port eth1 --radio wiphy0 --num_stations $NUM_STA --security $SECURITY --ssid jedway-wpa2-x2048-4-4 --passwd jedway-wpa2-x2048-4-4 --test_duration 3m" + "./test_ipv4_l4.py --radio wiphy3 --num_stations 4 --security wpa2 --ssid jedway-wpa2-x2048-4-1 --passwd jedway-wpa2-x2048-4-1 --url \"dl http://10.40.0.1 /dev/null\" --test_duration 2m --debug" ) -function ret_case_num(){ +function ret_case_num() { case $1 in - "example_wpa_connection") - echo 1 ;; - "example_wpa2_connection") - echo 2 ;; - "example_wpa3_connection") - echo 4 ;; - "example_wep_connection") - echo 3 ;; - "test_ipv4_connection") - echo 5 ;; - "test_generic") - echo 6 ;; - "test_ipv4_l4_urls_per_ten") - echo 7 ;; - "test_ipv4_l4_wifi") - echo 8 ;; - "test_ipv4_l4") - echo 9 ;; + "example_wpa_connection") + echo 1 + ;; + "example_wpa2_connection") + echo 2 + ;; + "example_wpa3_connection") + echo 4 + ;; + "example_wep_connection") + echo 3 + ;; + "test_ipv4_connection") + echo 5 + ;; + "test_generic") + echo 6 + ;; + "test_ipv4_l4_urls_per_ten") + echo 7 + ;; + "test_ipv4_l4_wifi") + echo 8 + ;; + "test_ipv4_l4") + echo 9 + ;; esac } function blank_db() { - echo "Loading blank scenario..." >> ~/test_all_output_file.txt - ./scenario.py --load BLANK >> ~/test_all_output_file.txt - #check_blank.py + echo "Loading blank scenario..." >>~/test_all_output_file.txt + ./scenario.py --load BLANK >>~/test_all_output_file.txt + #check_blank.py } -function echo_print(){ - echo "Beginning $CURR_TEST_NAME test..." >> ~/test_all_output_file.txt +function echo_print() { + echo "Beginning $CURR_TEST_NAME test..." >>~/test_all_output_file.txt } -function run_test(){ - for i in "${testCommands[@]}"; do - CURR_TEST_NAME=${i%%.py*} - CURR_TEST_NAME=${CURR_TEST_NAME#./*} - CURR_TEST_NUM=$(ret_case_num $CURR_TEST_NAME) - if [[ $CURR_TEST_NUM -gt $STOP_NUM ]] || [[ $STOP_NUM -eq $CURR_NUM && $STOP_NUM -ne 0 ]]; then - exit 1 - fi - if [[ $CURR_TEST_NUM -gt $START_NUM ]] || [[ $CURR_TEST_NUM -eq $START_NUM ]]; then - echo_print - eval $i >> ~/test_all_output_file.txt - if [ $? -ne 0 ]; then +function run_test() { + for i in "${testCommands[@]}"; do + CURR_TEST_NAME=${i%%.py*} + CURR_TEST_NAME=${CURR_TEST_NAME#./*} + CURR_TEST_NUM=$(ret_case_num $CURR_TEST_NAME) + if [[ $CURR_TEST_NUM -gt $STOP_NUM ]] || [[ $STOP_NUM -eq $CURR_NUM && $STOP_NUM -ne 0 ]]; then + exit 1 + fi + if [[ $CURR_TEST_NUM -gt $START_NUM ]] || [[ $CURR_TEST_NUM -eq $START_NUM ]]; then + echo_print + eval $i >>~/test_all_output_file.txt + if [ $? -ne 0 ]; then echo $CURR_TEST_NAME failure - else - echo $CURR_TEST_NAME success + else + echo $CURR_TEST_NAME success fi - if [[ "${CURR_TEST_NAME}" = @(example_wpa_connection|example_wpa2_connection|example_wpa3_connection|example_wep_connection) ]]; then - blank_db - fi - fi + if [[ "${CURR_TEST_NAME}" = @(example_wpa_connection|example_wpa2_connection|example_wpa3_connection|example_wep_connection) ]]; then + blank_db + fi + fi done } -function check_args(){ +function check_args() { if [ ! -z $1 ]; then START_NUM=$1 fi if [ ! -z $2 ]; then STOP_NUM=$2 - fi + fi } -true > ~/test_all_output_file.txt -check_args $1 $2 +true >~/test_all_output_file.txt +check_args $1 $2 run_test -#test generic and fileio are for macvlans +#test generic and fileio are for macvlans From d4a2eca208a37891d80091ff6a8b98247eab3322 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 11:52:33 -0800 Subject: [PATCH 24/35] test_all_scripts.sh: replaces case with associative array, replaces string tests with arithmetic tests --- py-scripts/test_all_scripts.sh | 64 ++++++++++++++-------------------- 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/py-scripts/test_all_scripts.sh b/py-scripts/test_all_scripts.sh index 1d277311..66a19978 100755 --- a/py-scripts/test_all_scripts.sh +++ b/py-scripts/test_all_scripts.sh @@ -8,10 +8,12 @@ SSID_USED="jedway-wpa2-x2048-5-3" PASSWD_USED="jedway-wpa2-x2048-5-3" RADIO_USED="wiphy1" SECURITY="wpa2" -CURR_TEST_NAME="BLANK" -CURR_TEST_NUM=0 -STOP_NUM=9 + START_NUM=0 +CURR_TEST_NUM=0 +CURR_TEST_NAME="BLANK" +STOP_NUM=9 + #Test array testCommands=("./example_wpa_connection.py --num_stations $NUM_STA --ssid jedway-r8000-36 --passwd jedway-r8000-36 --radio $RADIO_USED --security wpa" "./example_wpa2_connection.py --num_stations $NUM_STA --ssid $SSID_USED --passwd $SSID_USED --radio $RADIO_USED --security wpa2" @@ -24,37 +26,19 @@ testCommands=("./example_wpa_connection.py --num_stations $NUM_STA --ssid jedway "./test_ipv4_l4_wifi.py --upstream_port eth1 --radio wiphy0 --num_stations $NUM_STA --security $SECURITY --ssid jedway-wpa2-x2048-4-4 --passwd jedway-wpa2-x2048-4-4 --test_duration 3m" "./test_ipv4_l4.py --radio wiphy3 --num_stations 4 --security wpa2 --ssid jedway-wpa2-x2048-4-1 --passwd jedway-wpa2-x2048-4-1 --url \"dl http://10.40.0.1 /dev/null\" --test_duration 2m --debug" ) -function ret_case_num() { - case $1 in - "example_wpa_connection") - echo 1 - ;; - "example_wpa2_connection") - echo 2 - ;; - "example_wpa3_connection") - echo 4 - ;; - "example_wep_connection") - echo 3 - ;; - "test_ipv4_connection") - echo 5 - ;; - "test_generic") - echo 6 - ;; - "test_ipv4_l4_urls_per_ten") - echo 7 - ;; - "test_ipv4_l4_wifi") - echo 8 - ;; - "test_ipv4_l4") - echo 9 - ;; - esac -} +declare -A name_to_num +name_to_num=( + ["example_wpa_connection"]=1 + ["example_wpa2_connection"]=2 + ["example_wpa3_connection"]=4 + ["example_wep_connection"]=3 + ["test_ipv4_connection"]=5 + ["test_generic"]=6 + ["test_ipv4_l4_urls_per_ten"]=7 + ["test_ipv4_l4_wifi"]=8 + ["test_ipv4_l4"]=9 +) + function blank_db() { echo "Loading blank scenario..." >>~/test_all_output_file.txt ./scenario.py --load BLANK >>~/test_all_output_file.txt @@ -65,14 +49,20 @@ function echo_print() { } function run_test() { for i in "${testCommands[@]}"; do + set -x + set -e CURR_TEST_NAME=${i%%.py*} CURR_TEST_NAME=${CURR_TEST_NAME#./*} - CURR_TEST_NUM=$(ret_case_num $CURR_TEST_NAME) - if [[ $CURR_TEST_NUM -gt $STOP_NUM ]] || [[ $STOP_NUM -eq $CURR_NUM && $STOP_NUM -ne 0 ]]; then + CURR_TEST_NUM="${name_to_num[$CURR_TEST_NAME]}" + echo "$CURR_TEST_NAME $CURR_TEST_NUM" + + if (( $CURR_TEST_NUM > $STOP_NUM )) || (( $STOP_NUM == $CURR_TEST_NUM )) && (( $STOP_NUM != 0 )); then exit 1 fi - if [[ $CURR_TEST_NUM -gt $START_NUM ]] || [[ $CURR_TEST_NUM -eq $START_NUM ]]; then + if (( $CURR_TEST_NUM > $START_NUM )) || (( $CURR_TEST_NUM == $START_NUM )); then echo_print + echo "$i" + sleep 3 eval $i >>~/test_all_output_file.txt if [ $? -ne 0 ]; then echo $CURR_TEST_NAME failure From e74dff7d9ba4f61d334c465b3c8f466d9b0a583a Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 11:55:51 -0800 Subject: [PATCH 25/35] test_all_scripts.sh: adds test for debug env variable --- py-scripts/test_all_scripts.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/py-scripts/test_all_scripts.sh b/py-scripts/test_all_scripts.sh index 66a19978..7957865c 100755 --- a/py-scripts/test_all_scripts.sh +++ b/py-scripts/test_all_scripts.sh @@ -49,8 +49,6 @@ function echo_print() { } function run_test() { for i in "${testCommands[@]}"; do - set -x - set -e CURR_TEST_NAME=${i%%.py*} CURR_TEST_NAME=${CURR_TEST_NAME#./*} CURR_TEST_NUM="${name_to_num[$CURR_TEST_NAME]}" @@ -62,10 +60,11 @@ function run_test() { if (( $CURR_TEST_NUM > $START_NUM )) || (( $CURR_TEST_NUM == $START_NUM )); then echo_print echo "$i" - sleep 3 + [[ x$DEBUG != x ]] && sleep 2 eval $i >>~/test_all_output_file.txt if [ $? -ne 0 ]; then echo $CURR_TEST_NAME failure + [[ x$DEBUG != x ]] && exit 1 else echo $CURR_TEST_NAME success fi From 033ed53e9074cf26cafea30732854baab18bb60b Mon Sep 17 00:00:00 2001 From: Dipti Date: Fri, 18 Dec 2020 15:10:41 -0800 Subject: [PATCH 26/35] consistency in output --- py-json/LANforge/lfcli_base.py | 8 ++++---- py-scripts/test_ipv4_variable_time.py | 27 ++++++--------------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/py-json/LANforge/lfcli_base.py b/py-json/LANforge/lfcli_base.py index ed8901ba..a280adcc 100644 --- a/py-json/LANforge/lfcli_base.py +++ b/py-json/LANforge/lfcli_base.py @@ -508,10 +508,10 @@ class LFCliBase: optional.add_argument('--proxy', nargs='?', default=None, help='Connection proxy like http://proxy.localnet:80 or https://user:pass@proxy.localnet:3128') #Required Args - required.add_argument('--radio', help='radio EID, e.g: 1.wiphy2', required=True) - required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True) - required.add_argument('--ssid', help='SSID for stations to associate to', required=True) - required.add_argument('--passwd', '--password' ,'--key', help='WiFi passphrase/password/key', required=True) + required.add_argument('--radio', help='radio EID, e.g: 1.wiphy2') + required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >') + required.add_argument('--ssid', help='WiFi SSID for script objects to associate to') + required.add_argument('--passwd', '--password' ,'--key', help='WiFi passphrase/password/key') return parser diff --git a/py-scripts/test_ipv4_variable_time.py b/py-scripts/test_ipv4_variable_time.py index 5ed682a3..f69514fd 100755 --- a/py-scripts/test_ipv4_variable_time.py +++ b/py-scripts/test_ipv4_variable_time.py @@ -201,26 +201,13 @@ Create stations to test connection and traffic on VAPs of varying security types description='''\ test_ipv4_variable_time.py: --------------------- -Options: --upstream_port eth1 +------------------------------------------------------------ +Options: + --upstream_port eth1 --radio wiphy0 --num_stations 32 - --security {open|wep|wpa|wpa2|wpa3} \\ + --security {open|wep|wpa|wpa2|wpa3} --mode 1 - {"auto" : "0", - "a" : "1", - "b" : "2", - "g" : "3", - "abg" : "4", - "abgn" : "5", - "bgn" : "6", - "bg" : "7", - "abgnAC" : "8", - "anAC" : "9", - "an" : "10", - "bgnAC" : "11", - "abgnAX" : "12", - "bgnAX" : "13", --ssid netgear --password admin123 --test_duration 2m (default) @@ -229,13 +216,11 @@ Options: --upstream_port eth1 --ap "00:0e:8e:78:e1:76" --debug''') - #optional = parser.add_argument_group('optional arguments') - #required = parser.add_argument_group('required arguments') required = None for agroup in parser._action_groups: if agroup.title == "required arguments": required = agroup - # pprint.pprint(required) + if required is not None: required.add_argument('--a_min', help='bps rate minimum for endpoint A', type=int, default=256000) required.add_argument('--b_min', help='bps rate minimum for endpoint B', type=int, default=256000) @@ -245,7 +230,7 @@ Options: --upstream_port eth1 for agroup in parser._action_groups: if agroup.title == "optional arguments": optional = agroup - # pprint.pprint(optional) + if optional is not None: optional.add_argument("--ap", help="Add BSSID of access point to connect to") optional.add_argument('--mode', help=LFCliBase.Help_Mode) From 1af13e32a5b7d9924348f34e66a969973507f39e Mon Sep 17 00:00:00 2001 From: Dipti Date: Fri, 18 Dec 2020 15:17:58 -0800 Subject: [PATCH 27/35] addition of new argparse functions to help --- py-scripts/test_ipv4_variable_time.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/py-scripts/test_ipv4_variable_time.py b/py-scripts/test_ipv4_variable_time.py index f69514fd..33305f8e 100755 --- a/py-scripts/test_ipv4_variable_time.py +++ b/py-scripts/test_ipv4_variable_time.py @@ -202,8 +202,10 @@ Create stations to test connection and traffic on VAPs of varying security types description='''\ test_ipv4_variable_time.py: ------------------------------------------------------------ -Options: +Options: --mgr localhost + --mgr_port 4122 --upstream_port eth1 + --proxy --radio wiphy0 --num_stations 32 --security {open|wep|wpa|wpa2|wpa3} @@ -213,6 +215,10 @@ Options: --test_duration 2m (default) --a_min 1000 --b_min 1000 + --monitor + --monitor_interval_sec 6 + --save_to + --test_id --ap "00:0e:8e:78:e1:76" --debug''') From 945f5fbb5b2f97f3044e88600610284c2bfb9d32 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 15:27:25 -0800 Subject: [PATCH 28/35] check_large_files: menu processing and survey methods, WIP --- check_large_files.bash | 173 +++++++++++++++++++++++++++++++++++------ 1 file changed, 151 insertions(+), 22 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index e23ffa81..9adb595b 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -27,6 +27,19 @@ if (( eyedee != 0 )); then exit 1 fi +function contains () { + if [[ x$1 = x ]] || [[ x$2 = x ]]; then + echo "contains wants ARRAY and ITEM arguments: if contains name joe; then... }$" + exit 1 + fi + for item in "${$1[@]}"; do + echo $item + [[ "$2" = "$item" ]] && return 0 + done + return 1 +} + + # these are default selections selections=() deletion_targets=() @@ -71,7 +84,6 @@ while getopts $opts opt; do q) quiet=1 verbose=0 - selections+=($opt) ;; t) selections+=($opt) @@ -79,7 +91,6 @@ while getopts $opts opt; do v) quiet=0 verbose=1 - selections+=($opt) ;; *) echo "unknown option: $opt" @@ -99,27 +110,96 @@ function hr() { echo "$HR" } +declare -A totals=( + [b]=0 + [c]=0 + [d]=0 + [k]=0 + [l]=0 + [m]=0 + [r]=0 + [t]=0 +) +declare -A surveyors_map=( + [b]="survey_kernel_files" + [c]="survey_core_files" + [d]="survey_lf_downloads" + [k]="survey_ath10_files" + [l]="survey_var_log" + [m]="survey_mnt_lf_files" + [r]="survey_report_data" + [t]="survey_var_tmp" +) + +declare -A cleaners_map=( + [b]="clean_old_kernels" + [c]="clean_core_files" + [d]="clean_lf_downloads" + [k]="clean_ath10_files" + [l]="clean_var_log" + [m]="clean_mnt_lf_files" + [r]="compress_report_data" + [t]="clean_var_tmp" +) + +kernel_files=() +survey_kernel_files() { + mapfile -t kernel_files < <(ls /boot/* /lib/modules/* 2>/dev/null) + totals[b]=$(du -hc "$kernel_files" | awk '/total/{print $1}') +} + # Find core files core_files=() -cd / -mapfile -t core_files < <(ls /core* /home/lanforge/core* 2>/dev/null) +survey_core_files() { + cd / + mapfile -t core_files < <(ls /core* /home/lanforge/core* 2>/dev/null) + totals[c]=$(du -hc "${core_files[@]}" | awk '/total/{print $1}') +} + +# downloads +downloads=() +survey_lf_downloads() { + cd /home/lanforge/Downloads || return 1 + mapfile -t downloads < <(ls *gz *z2 *-Installer.exe *firmware* kinst_* *Docs* 2>/dev/null) + totals[d]=$(du -hc "${downloads[@]}" | awk '/total/{print $1}') +} # Find ath10k crash residue ath10_files=() -mapfile -t ath10_files < <(ls /home/lanforge/ath10* 2>/dev/null) +survey_ath10_files() { + mapfile -t ath10_files < <(ls /home/lanforge/ath10* 2>/dev/null) + totals[k]=$(du -sh "${ath10_files}" 2>/dev/null) +} + +# stuff in var log +var_log_files=() +survey_var_log() { + mapfile -t var_log_files < <(ls /var/log/* 2>/dev/null) + totals[l]=$(du -sh "${var_log_files}" 2>/dev/null) +} # Find size of /mnt/lf that is not mounted -cd /mnt -usage_mnt=`du -shxc .` +mnt_lf_files=() +survey_mnt_lf_files() { + [ ! -d /mnt/lf ] && return 0 + mapfile -t mnt_lf_files < <(find /mnt/lf -type f --one_filesystem) + totals[m]=$(du -xhc "${mnt_lf_files[@]}") +} -# Find size of /lib/modules -cd /lib/modules -mapfile -t usage_libmod < <(du -sh *) +## Find size of /lib/modules +# cd /lib/modules +# mapfile -t usage_libmod < <(du -sh *) # Find how many kernels are installed -cd /boot -mapfile -t boot_kernels < <(ls init*) -boot_usage=`du -sh .` +# cd /boot +# mapfile -t boot_kernels < <(ls init*) +# boot_usage=`du -sh .` + +report_files=() +survey_report_data() { + cd /home/lanforge + totals=that +} # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # @@ -152,15 +232,64 @@ sleep 1 # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # ask to remove if we are interactive # # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # -if -item="" -while [[ $item != q ]]; do + + +choice="" +while [[ $choice != q ]]; do hr + #abcdhklmqrtv echo "Would you like to delete? " - echo " 1) core crash files" - echo " 2) ath10k crash files" - echo " 3) old var/www downloads" - echo " 4) old lanforge downloads" - echo " 5) orphaned /mnt/lf files" - read -p "[1-5] or q ? " item + echo " b) old kernels" + echo " c) core crash files" + echo " d) old LANforge downloads" + echo " k) ath10k crash files" + echo " l) old /var/log files" + echo " m) orphaned /mnt/lf files" + echo " r) compress .csv report files" + echo " t) clean /var/tmp" + read -p "[1-5] or q ? " choice + + case "$choice" in + b ) + printf "%s\n" "${kernels[@]}" + clean_old_kernels + ;; + c ) + printf "%s\n" "${core_files[@]}" + clean_core_files + ;; + d ) + printf "%s\n" "${lf_downloads[@]}" + clean_lf_downloads + ;; + k ) + printf "%s\n" "${ath10_files[@]}" + clean_ath10_files + ;; + l ) + printf "%s\n" "${var_log_files[@]}" + clean_var_log + ;; + m ) + printf "%s\n" "${mnt_lf_files[@]}" + clean_mnt_lf_files + ;; + r ) + printf "%s\n" "${report_data_dirs[@]}" + compress_report_data + ;; + t ) + printf "%s\n" "${var_tmp_files[@]}" + clean_var_tmp + ;; + q ) + break + ;; + * ) + echo "not an option [$choice]" + ;; + esac done + + +echo bye \ No newline at end of file From 549c8777b24d6f250ab223cf3ff54a5bacf240cb Mon Sep 17 00:00:00 2001 From: Dipti Date: Fri, 18 Dec 2020 15:35:00 -0800 Subject: [PATCH 29/35] argparse and help fixed --- py-scripts/example_security_connection.py | 52 ++++++++++------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/py-scripts/example_security_connection.py b/py-scripts/example_security_connection.py index c07ec9ff..6c30c600 100755 --- a/py-scripts/example_security_connection.py +++ b/py-scripts/example_security_connection.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 + import sys import os if sys.version_info[0] != 3: @@ -76,49 +77,37 @@ def main(): prog='example_security_connection.py', formatter_class=argparse.RawTextHelpFormatter, epilog='''\ - Example flags and command line input to run the script. + This python script creates an inputted number of stations using user-inputted security. This verifies that the most basic form of security works with the LANforge device. ''', - description='''\ example_security_connection.py - -------------------- - This python script creates an inputted number of stations using user-inputted security. This verifies that the most basic form of security works with the LANforge device. - -------------------- + --------------------------------------------------------------------------- - Generic command example: - python3 ./example_security_connection.py + Example of command line to run: + ./example_security_connection.py --mgr localhost --mgr_port 8080 --num_stations 6 --mode 1 - {"auto" : "0", - "a" : "1", - "b" : "2", - "g" : "3", - "abg" : "4", - "abgn" : "5", - "bgn" : "6", - "bg" : "7", - "abgnAC" : "8", - "anAC" : "9", - "an" : "10", - "bgnAC" : "11", - "abgnAX" : "12", - "bgnAX" : "13", - "anAX" : "14"} --radio wiphy2 --security {open|wep|wpa|wpa2|wpa3} --ssid netgear-wpa3 --ap "00:0e:8e:78:e1:76" --passwd admin123-wpa3 --debug - ''') - optional = parser.add_argument_group('optional arguments') - required = parser.add_argument_group('required arguments') - required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True) - optional.add_argument('--mode',help='Used to force mode of stations') - optional.add_argument('--ap',help='Used to force a connection to a particular AP') + required=None + for agroup in parser._action_groups: + if agroup.title == "required arguments": + required = agroup + #if required is not None: + optional = None + for agroup in parser._action_groups: + if agroup.title == "optional arguments": + optional = agroup + if optional is not None: + optional.add_argument('--mode',help=LFCliBase.Help_Mode) + optional.add_argument('--ap',help='Add BSSID of access point to connect to') args = parser.parse_args() num_sta = 2 @@ -131,8 +120,11 @@ def main(): end_id_=num_sta-1, padding_number_=10000, radio=args.radio) - ip_test = IPv4Test(host=args.mgr, port=args.mgr_port, ssid=args.ssid, password=args.passwd, radio=args.radio, mode= args.mode, - security=args.security, sta_list=station_list, ap=args.ap) + ip_test = IPv4Test(host=args.mgr, port=args.mgr_port, + ssid=args.ssid, password=args.passwd, + radio=args.radio, mode= args.mode, + security=args.security, sta_list=station_list, + ap=args.ap) ip_test.cleanup(station_list) ip_test.timeout = 60 ip_test.build() From 4193b54e5c302c8fd4d6b63b8aad1b881fda8eb7 Mon Sep 17 00:00:00 2001 From: Dipti Date: Fri, 18 Dec 2020 16:06:08 -0800 Subject: [PATCH 30/35] fixed help, added mode and AP customization --- py-scripts/test_ipv6_connection.py | 62 ++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/py-scripts/test_ipv6_connection.py b/py-scripts/test_ipv6_connection.py index 79fadcb9..bfea8ccc 100755 --- a/py-scripts/test_ipv6_connection.py +++ b/py-scripts/test_ipv6_connection.py @@ -18,7 +18,7 @@ import pprint class IPv6Test(LFCliBase): - def __init__(self, ssid, security, password, sta_list=None, num_stations=0, prefix="00000", host="localhost", port=8080, + def __init__(self, ssid, security, password,ap=None, mode=0, sta_list=None, num_stations=0, prefix="00000", host="localhost", port=8080, _debug_on=False, timeout=120, radio="wiphy0", _exit_on_error=False, _exit_on_fail=False, @@ -30,6 +30,8 @@ class IPv6Test(LFCliBase): self.radio = radio self.security = security self.password = password + self.ap=ap + self.mode=mode self.num_stations = num_stations self.sta_list = sta_list self.timeout = timeout @@ -41,12 +43,16 @@ class IPv6Test(LFCliBase): self.station_profile.lfclient_url = self.lfclient_url self.station_profile.ssid = self.ssid - self.station_profile.ssid_pass = self.password, + self.station_profile.ssid_pass = self.password + if mode is not None: + self.station_profile.mode = mode self.station_profile.security = self.security self.station_profile.number_template_ = self.number_template - self.station_profile.mode = 0 - def build(self): + + def build(self): + if self.ap is not None: + self.station_profile.set_command_param("add_sta", "ap", self.ap) self.station_profile.use_security(self.security, self.ssid, self.password) self.station_profile.set_number_template(self.prefix) print("Creating stations") @@ -104,7 +110,6 @@ class IPv6Test(LFCliBase): return self.passes() def stop(self): - # Bring stations down self.station_profile.admin_down() def cleanup(self, sta_list): @@ -125,26 +130,39 @@ def main(): description='''\ test_ipv6_connection.py: --------------------- +-------------------------------------------------- Generic command example: -python3 ./test_ipv6_connection.py --upstream_port eth1 \\ - --radio wiphy0 \\ - --num_stations 3 \\ - --security {open|wep|wpa|wpa2|wpa3} \\ - --ssid netgear \\ - --passwd admin123 \\ - --dest 10.40.0.1 \\ - --test_duration 2m \\ - --interval 1s \\ - -- timeout 120 \\ - --debug +python3 ./test_ipv6_connection.py + --upstream_port eth1 + --radio wiphy0 + --num_stations 3 + --proxy + --security {open|wep|wpa|wpa2|wpa3} + --ssid netgear + --passwd admin123 + --mode 1 + --ap "00:0e:8e:78:e1:76" + --test_id + -- timeout 120 + --debug ''') - optional = parser.add_argument_group('optional arguments') - required = parser.add_argument_group('required arguments') - required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True) - parser.add_argument('--timeout', help='--timeout sets the length of time to wait until a connection is successful', default=30) + required = None + for agroup in parser._action_groups: + if agroup.title == "required arguments": + required = agroup + #if required is not None: + + optional = None + for agroup in parser._action_groups: + if agroup.title == "optional arguments": + optional = agroup + + if optional is not None: + optional.add_argument("--ap", help="Add BSSID of access point to connect to") + optional.add_argument('--mode', help=LFCliBase.Help_Mode) + optional.add_argument('--timeout', help='--timeout sets the length of time to wait until a connection is successful', default=30) args = parser.parse_args() num_sta=2 @@ -159,6 +177,8 @@ python3 ./test_ipv6_connection.py --upstream_port eth1 \\ ssid=args.ssid, password=args.passwd, security=args.security, + ap=args.ap, + mode=args.mode, sta_list=station_list, _debug_on=args.debug) ipv6_test.cleanup(station_list) From e11cb78bd1aae65a5bd6d93cbef62da658bb090c Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 16:56:32 -0800 Subject: [PATCH 31/35] check_large_files.bash: adds lots of disk size logic, WIP --- check_large_files.bash | 92 ++++++++++++++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 21 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index 9adb595b..ea0b8542 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -32,14 +32,14 @@ function contains () { echo "contains wants ARRAY and ITEM arguments: if contains name joe; then... }$" exit 1 fi - for item in "${$1[@]}"; do + local zarray="${1}[@]" + for item in "${zarray[@]}"; do echo $item [[ "$2" = "$item" ]] && return 0 done return 1 } - # these are default selections selections=() deletion_targets=() @@ -120,6 +120,16 @@ declare -A totals=( [r]=0 [t]=0 ) +declare -A desc=( + [b]="kernel files" + [c]="core files" + [d]="lf downloads" + [k]="lf/ath10 files" + [l]="/var/log" + [m]="/mnt/lf files" + [r]="lf/report_data" + [t]="/var/tmp" +) declare -A surveyors_map=( [b]="survey_kernel_files" [c]="survey_core_files" @@ -154,6 +164,7 @@ survey_core_files() { cd / mapfile -t core_files < <(ls /core* /home/lanforge/core* 2>/dev/null) totals[c]=$(du -hc "${core_files[@]}" | awk '/total/{print $1}') + [[ x${totals[c]} = x ]] && totals[c]=0 } # downloads @@ -162,20 +173,31 @@ survey_lf_downloads() { cd /home/lanforge/Downloads || return 1 mapfile -t downloads < <(ls *gz *z2 *-Installer.exe *firmware* kinst_* *Docs* 2>/dev/null) totals[d]=$(du -hc "${downloads[@]}" | awk '/total/{print $1}') + [[ x${totals[d]} = x ]] && totals[d]=0 } # Find ath10k crash residue ath10_files=() survey_ath10_files() { mapfile -t ath10_files < <(ls /home/lanforge/ath10* 2>/dev/null) - totals[k]=$(du -sh "${ath10_files}" 2>/dev/null) + totals[k]=$(du -hc "${ath10_files}" 2>/dev/null | awk '/total/{print $1}') + [[ x${totals[k]} = x ]] && totals[k]=0 } # stuff in var log var_log_files=() survey_var_log() { - mapfile -t var_log_files < <(ls /var/log/* 2>/dev/null) - totals[l]=$(du -sh "${var_log_files}" 2>/dev/null) + mapfile -t var_log_files < <(find /var/log -type f -size +10M 2>/dev/null) + totals[l]=$(du -hc "${var_log_files}" 2>/dev/null | awk '/total/{print $1}' ) + [[ x${totals[l]} = x ]] && totals[l]=0 +} + +# stuff in var tmp +var_tmp_files=() +survey_var_tmp() { + mapfile -t var_tmp_files < <(find /var/tmp -type f 2>/dev/null) + totals[t]=$(du -sh "${var_tmp_files}" 2>/dev/null | awk '/total/{print $1}' ) + [[ x${totals[t]} = x ]] && totals[t]=0 } # Find size of /mnt/lf that is not mounted @@ -183,7 +205,8 @@ mnt_lf_files=() survey_mnt_lf_files() { [ ! -d /mnt/lf ] && return 0 mapfile -t mnt_lf_files < <(find /mnt/lf -type f --one_filesystem) - totals[m]=$(du -xhc "${mnt_lf_files[@]}") + totals[m]=$(du -xhc "${mnt_lf_files[@]}" 2>/dev/null | awk '/total/{print $1}') + [[ x${totals[m]} = x ]] && totals[m]=0 } ## Find size of /lib/modules @@ -202,26 +225,47 @@ survey_report_data() { } +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # +# gather usage areas +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # +survey_areas() { + local area + echo -n "surveying..." + for area in "${!surveyors_map[@]}"; do + echo -n "#" + ${surveyors_map[$area]} + done + echo "" +} + # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # report sizes here # # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # -if (( ${#core_files[@]} > 0 )); then - echo "Core Files:" - hr - printf ' %s\n' "${core_files[@]}" - hr -fi +disk_usage_report() { + for k in "${!totals[@]}"; do + echo -e "\t${desc[$k]}:\t${totals[$k]}" + done +} +survey_areas +disk_usage_report +exit +#if (( ${#core_files[@]} > 0 )); then +# echo "Core Files:" +# hr +# printf ' %s\n' "${core_files[@]}" +# hr +#fi -echo "Usage of /mnt: $usage_mnt" -echo "Usage of /lib/modules: $usage_libmod" -echo "Boot usage: $boot_usage" +#echo "Usage of /mnt: $usage_mnt" +#echo "Usage of /lib/modules: $usage_libmod" +#echo "Boot usage: $boot_usage" -if (( ${#boot_kernels[@]} > 1 )); then - echo "Boot ramdisks:" - hr - printf ' %s\n' "${boot_kernels[@]}" - hr -fi +#if (( ${#boot_kernels[@]} > 1 )); then +# echo "Boot ramdisks:" +# hr +# printf ' %s\n' "${boot_kernels[@]}" +# hr +#fi # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # delete extra things now # @@ -233,6 +277,12 @@ sleep 1 # ask to remove if we are interactive # # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # +if contains "selections" "a" ; then + for z in "${selections[@]}"; do + echo "will execute $z" + done + exit 0 +fi choice="" while [[ $choice != q ]]; do From 547736836921aeb53774b35bacbf12e901a087b1 Mon Sep 17 00:00:00 2001 From: Logan Lipke Date: Fri, 18 Dec 2020 15:39:35 -0800 Subject: [PATCH 32/35] Added check for port_list in wait_until_ports_disappear to ensure list is being used --- py-json/LANforge/LFUtils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/py-json/LANforge/LFUtils.py b/py-json/LANforge/LFUtils.py index 62766311..088b01f8 100644 --- a/py-json/LANforge/LFUtils.py +++ b/py-json/LANforge/LFUtils.py @@ -445,7 +445,10 @@ def waitUntilPortsDisappear(base_url="http://localhost:8080", port_list=[], debu def wait_until_ports_disappear(base_url="http://localhost:8080", port_list=[], debug=False): print("Waiting until ports disappear...") url = "/port/1" - found_stations = port_list.copy() + if isinstance(port_list, list): + found_stations = port_list.copy() + else: + found_stations = [port_list] while len(found_stations) > 0: found_stations = [] From fa4d51907886e10f97dc22081d8aa5496d0f7495 Mon Sep 17 00:00:00 2001 From: Logan Lipke Date: Fri, 18 Dec 2020 17:00:33 -0800 Subject: [PATCH 33/35] Clarified help output, added more command examples --- py-scripts/test_fileio.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/py-scripts/test_fileio.py b/py-scripts/test_fileio.py index 439b0032..12f4f974 100755 --- a/py-scripts/test_fileio.py +++ b/py-scripts/test_fileio.py @@ -564,11 +564,24 @@ def main(): test_fileio.py: -------------------- Generic command layout: -./test_fileio.py --macvlan_parent --num_ports --use_macvlans +./test_fileio.py --macvlan_parent --num_ports --use_macvlans --first_mvlan_ip --netmask --gateway ./test_fileio.py --macvlan_parent eth2 --num_ports 3 --use_macvlans --first_mvlan_ip 192.168.92.13 --netmask 255.255.255.0 --gateway 192.168.92.1 + +./test_fileio.py --radio 1.wiphy0 --test_duration 1m --macvlan_parent eth1 --num_ports 3 --use_macvlans + --use_ports eth1#0,eth1#1,eth1#2 --connections_per_port 2 --mode write + +./test_fileio.py --radio 1.wiphy0 --test_duration 1m --macvlan_parent eth1 --num_ports 3 --use_macvlans + --first_mvlan_ip 10.40.3.100 --netmask 255.255.240.0 --gateway 10.40.0.1 + --use_test_groups --write_only_test_group test_wo --read_only_test_group test_ro + --add_to_group test_wo --cxs test_wo0000,test_wo0001,test_wo0002 + +./test_fileio.py --radio 1.wiphy0 --test_duration 1m --macvlan_parent eth1 --num_ports 3 --use_macvlans + --use_ports eth1#0=10.40.3.103,eth1#1,eth1#2 --connections_per_port 2 + --netmask 255.255.240.0 --gateway 10.40.0.1 + ''') parser.add_argument('--num_stations', help='Number of stations to create', default=0) parser.add_argument('--radio', help='radio EID, e.g: 1.wiphy2') @@ -599,7 +612,8 @@ Generic command layout: parser.add_argument('--connections_per_port', help='specifies number of connections to be used per port', default=1, type=int) parser.add_argument('--use_ports', help='list of comma separated ports to use with ips, \'=\' separates name and ip' - '{ port_name1=ip_addr1,port_name1=ip_addr2 }', default=None) + '{ port_name1=ip_addr1,port_name1=ip_addr2 }. ' + 'Ports without ips will be left alone', default=None) parser.add_argument('--use_macvlans', help='will create macvlans', action='store_true', default=False) parser.add_argument('--first_mvlan_ip', help='specifies first static ip address to be used or dhcp', default=None) parser.add_argument('--netmask', help='specifies netmask to be used with static ip addresses', default=None) From 1923fc5c024fe7a4a27c9bd9b37c554b73f446fc Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Sun, 20 Dec 2020 13:12:08 -0800 Subject: [PATCH 34/35] check_large_files: making progress --- check_large_files.bash | 235 ++++++++++++++++++++++++++++++++--------- 1 file changed, 185 insertions(+), 50 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index ea0b8542..2dcab408 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -27,13 +27,31 @@ if (( eyedee != 0 )); then exit 1 fi +debug() { + if [[ x$verbose = x ]] || (( $verbose < 1 )); then return; fi + + echo ": $1" +} + +note() { + #set -x + if (( $quiet > 0 )); then return; fi + + echo "# $1" + #set +x +} + function contains () { if [[ x$1 = x ]] || [[ x$2 = x ]]; then echo "contains wants ARRAY and ITEM arguments: if contains name joe; then... }$" exit 1 fi - local zarray="${1}[@]" - for item in "${zarray[@]}"; do + local tmp=$1[@] + local array=( "${!tmp[@]}" ) + if (( ${#array[@]} < 1 )); then + return 1 + fi + for item in "${array[@]}"; do echo $item [[ "$2" = "$item" ]] && return 0 done @@ -100,10 +118,10 @@ while getopts $opts opt; do esac done -if (( ${#selections} < 1 )); then - echo "$USAGE" - exit 0 -fi +#if (( ${#selections} < 1 )); then +# echo "$USAGE" +# exit 0 +#fi HR=" ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----" function hr() { @@ -152,8 +170,84 @@ declare -A cleaners_map=( [t]="clean_var_tmp" ) +clean_old_kernels() { + note "Cleaning old kernels WIP" + kernels=() + # need to avoid most recent fedora kernel + if [ -x /usr/sbin/rpm ]; then + local kern_pkgs=( $( rpm -qa 'kernel*' | sort ) ) + local pkg + for pkg in "${kern_pkgs[@]}"; do + if [[ $pkg = kernel-tools* ]] \ + || [[ $pkg = kernel-headers* ]] \ + || [[ $pkg = kernel-devel* ]] ; then + continue + fi + kernels+=( $pkg ) + kernel_series=${pkg##kernel*5} + echo "K SER: $kernel_series" + done + fi + if (( $verbose > 0 )); then + printf "Would remove %s\n" "${kernels[@]}" + fi +} + +clean_core_files() { + note "Cleaning core files WIP" + if (( $verbose > 0 )); then + printf "%s\n" "${core_files[@]}" + fi +} + +clean_lf_downloads() { + note "Clean LF downloads WIP" + if (( $verbose > 0 )); then + printf "%s\n" "${lf_downloads[@]}" + fi +} + +clean_ath10_files() { + note "clean_ath10_files WIP" + if (( $verbose > 0 )); then + printf "%s\n" "${ath10_files[@]}" + fi +} + +clean_var_log() { + note "Clean var log WIP" + if (( $verbose > 0 )); then + printf "%s\n" "${var_log_files[@]}" + fi +} + +clean_mnt_fl_files() { + note "clean mnt lf files WIP" + if (( $verbose > 0 )); then + printf "%s\n" "${mnt_lf_files[@]}" + fi + +} + +compress_report_data() { + note "compress report data WIP" + if (( $verbose > 0 )); then + printf "%s\n" "${report_data_dirs[@]}" + fi +} + +clean_var_tmp() { + note "clean var tmp" + if (( $verbose > 0 )); then + printf "%s\n" "${var_tmp_files[@]}" + fi + rf -f "${var_tmp_files[@]}" +} + + kernel_files=() survey_kernel_files() { + debug "Surveying Kernel files" mapfile -t kernel_files < <(ls /boot/* /lib/modules/* 2>/dev/null) totals[b]=$(du -hc "$kernel_files" | awk '/total/{print $1}') } @@ -161,15 +255,24 @@ survey_kernel_files() { # Find core files core_files=() survey_core_files() { + debug "Surveying core files" cd / - mapfile -t core_files < <(ls /core* /home/lanforge/core* 2>/dev/null) - totals[c]=$(du -hc "${core_files[@]}" | awk '/total/{print $1}') + #set -x + mapfile -t core_files < <(ls /core* /home/lanforge/core* 2>/dev/null) 2>/dev/null + if [[ $verbose = 1 ]]; then + printf "%s\n" "${core_files[@]}" + fi + if (( ${#core_files[@]} > 0 )); then + totals[c]=$(du -hc "${core_files[@]}" | awk '/total/{print $1}') + fi + #set +x [[ x${totals[c]} = x ]] && totals[c]=0 } # downloads downloads=() survey_lf_downloads() { + debug "Surveying /home/lanforge downloads" cd /home/lanforge/Downloads || return 1 mapfile -t downloads < <(ls *gz *z2 *-Installer.exe *firmware* kinst_* *Docs* 2>/dev/null) totals[d]=$(du -hc "${downloads[@]}" | awk '/total/{print $1}') @@ -179,6 +282,7 @@ survey_lf_downloads() { # Find ath10k crash residue ath10_files=() survey_ath10_files() { + debug "Sureyinig ath10 crash files" mapfile -t ath10_files < <(ls /home/lanforge/ath10* 2>/dev/null) totals[k]=$(du -hc "${ath10_files}" 2>/dev/null | awk '/total/{print $1}') [[ x${totals[k]} = x ]] && totals[k]=0 @@ -187,6 +291,7 @@ survey_ath10_files() { # stuff in var log var_log_files=() survey_var_log() { + debug "Surveying var log" mapfile -t var_log_files < <(find /var/log -type f -size +10M 2>/dev/null) totals[l]=$(du -hc "${var_log_files}" 2>/dev/null | awk '/total/{print $1}' ) [[ x${totals[l]} = x ]] && totals[l]=0 @@ -195,16 +300,20 @@ survey_var_log() { # stuff in var tmp var_tmp_files=() survey_var_tmp() { + #set -x + debug "Surveying var tmp" mapfile -t var_tmp_files < <(find /var/tmp -type f 2>/dev/null) totals[t]=$(du -sh "${var_tmp_files}" 2>/dev/null | awk '/total/{print $1}' ) [[ x${totals[t]} = x ]] && totals[t]=0 + #set +x } # Find size of /mnt/lf that is not mounted mnt_lf_files=() survey_mnt_lf_files() { [ ! -d /mnt/lf ] && return 0 - mapfile -t mnt_lf_files < <(find /mnt/lf -type f --one_filesystem) + debug "Surveying mnt lf" + mapfile -t mnt_lf_files < <(find /mnt/lf -type f --one_filesystem 2>/dev/null) totals[m]=$(du -xhc "${mnt_lf_files[@]}" 2>/dev/null | awk '/total/{print $1}') [[ x${totals[m]} = x ]] && totals[m]=0 } @@ -220,8 +329,15 @@ survey_mnt_lf_files() { report_files=() survey_report_data() { + debug "Surveying for lanforge report data" cd /home/lanforge - totals=that + #set -x + local fnum=$( find -type f -name '*.csv' 2>/dev/null | wc -l ) + local fsiz=$( find -type f -name '*.csv' 2>/dev/null | xargs du -hc | awk '/total/{print $1}') + totals[r]="$fsiz" + [[ x${totals[r]} = x ]] && totals[r]=0 + report_files=("CSV files: $fnum tt $fsiz") + #set +x } @@ -230,9 +346,13 @@ survey_report_data() { # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # survey_areas() { local area - echo -n "surveying..." + if [[ x$quiet = x ]] || (( $quiet < 1 )); then + echo -n "Surveying..." + fi for area in "${!surveyors_map[@]}"; do - echo -n "#" + if [[ x$quiet = x ]] || (( $quiet < 1 )); then + echo -n "#" + fi ${surveyors_map[$area]} done echo "" @@ -248,13 +368,14 @@ disk_usage_report() { } survey_areas disk_usage_report -exit -#if (( ${#core_files[@]} > 0 )); then -# echo "Core Files:" -# hr -# printf ' %s\n' "${core_files[@]}" -# hr -#fi + +if (( ${#core_files[@]} > 0 )); then + note "Core Files detected, will remove:" + hr + printf ' %s\n' "${core_files[@]}" + hr + selections+=("c") +fi #echo "Usage of /mnt: $usage_mnt" #echo "Usage of /lib/modules: $usage_libmod" @@ -270,66 +391,78 @@ exit # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # delete extra things now # # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # -echo "Automatic deletion will include: " -echo " journalctl space" -sleep 1 -# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # -# ask to remove if we are interactive # -# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # - +#set -x if contains "selections" "a" ; then + note "Automatic deletion will include: " + printf "%s\n" "${selections[@]}" + debug "Doing automatic cleanup" for z in "${selections[@]}"; do - echo "will execute $z" + debug "Will perform ${desc[$z]}" + ${cleaners_map[$z]} done + + survey_areas + disk_usage_report + exit 0 fi +if (( ${#selections[@]} > 0 )) ; then + debug "Doing selected cleanup" + for z in "${selections[@]}"; do + debug "Will perform ${desc[$z]}" + ${cleaners_map[$z]} + selections=("${selections[@]/$z}") + done + + survey_areas + disk_usage_report +fi +set +x + +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # +# ask for things to remove if we are interactive # +# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # + + choice="" while [[ $choice != q ]]; do - hr - #abcdhklmqrtv - echo "Would you like to delete? " - echo " b) old kernels" - echo " c) core crash files" - echo " d) old LANforge downloads" - echo " k) ath10k crash files" - echo " l) old /var/log files" - echo " m) orphaned /mnt/lf files" - echo " r) compress .csv report files" - echo " t) clean /var/tmp" - read -p "[1-5] or q ? " choice + hr + #abcdhklmqrtv + echo "Would you like to delete? " + echo " b) old kernels ${totals[b]}" + echo " c) core crash files ${totals[c]}" + echo " d) old LANforge downloads ${totals[d]}" + echo " k) ath10k crash files ${totals[k]}" + echo " l) old /var/log files ${totals[l]}" + echo " m) orphaned /mnt/lf files ${totals[m]}" + echo " r) compress .csv report files ${totals[r]}" + echo " t) clean /var/tmp ${totals[t]}" + read -p "[1-5] or q ? " choice - case "$choice" in + case "$choice" in b ) - printf "%s\n" "${kernels[@]}" clean_old_kernels ;; c ) - printf "%s\n" "${core_files[@]}" clean_core_files ;; d ) - printf "%s\n" "${lf_downloads[@]}" clean_lf_downloads ;; k ) - printf "%s\n" "${ath10_files[@]}" clean_ath10_files ;; l ) - printf "%s\n" "${var_log_files[@]}" clean_var_log ;; m ) - printf "%s\n" "${mnt_lf_files[@]}" clean_mnt_lf_files ;; r ) - printf "%s\n" "${report_data_dirs[@]}" compress_report_data ;; t ) - printf "%s\n" "${var_tmp_files[@]}" clean_var_tmp ;; q ) @@ -338,8 +471,10 @@ while [[ $choice != q ]]; do * ) echo "not an option [$choice]" ;; - esac + esac + survey_areas + disk_usage_report done -echo bye \ No newline at end of file +echo bye From afc36e0d6d732c0b5a6e349f3037c1cc7c9bd5f4 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Mon, 21 Dec 2020 13:21:26 -0800 Subject: [PATCH 35/35] check_large_files.bash: fixes compares(), now sorts thru kernels better, shows corefiles, improves output --- check_large_files.bash | 94 ++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index 2dcab408..48abde69 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -4,6 +4,12 @@ # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # set -x # set -e +# these are default selections +selections=() +deletion_targets=() +show_menu=1 +verbose=0 +quiet=0 USAGE="$0 # Check for large files and purge many of the most inconsequencial -a # automatic: disable menu and clean automatically @@ -29,16 +35,12 @@ fi debug() { if [[ x$verbose = x ]] || (( $verbose < 1 )); then return; fi - echo ": $1" } note() { - #set -x if (( $quiet > 0 )); then return; fi - echo "# $1" - #set +x } function contains () { @@ -46,24 +48,24 @@ function contains () { echo "contains wants ARRAY and ITEM arguments: if contains name joe; then... }$" exit 1 fi - local tmp=$1[@] - local array=( "${!tmp[@]}" ) + # these two lines below are important to not modify + local tmp="${1}[@]" + local array=( ${!tmp} ) + + # if [[ x$verbose = x1 ]]; then + # printf "contains array %s\n" "${array[@]}" + # fi if (( ${#array[@]} < 1 )); then return 1 fi + local item for item in "${array[@]}"; do - echo $item + # debug "contains testing $2 == $item" [[ "$2" = "$item" ]] && return 0 done return 1 } -# these are default selections -selections=() -deletion_targets=() -show_menu=1 -verbose=0 -quiet=0 #opts="" opts="abcdhklmqrtv" while getopts $opts opt; do @@ -172,24 +174,46 @@ declare -A cleaners_map=( clean_old_kernels() { note "Cleaning old kernels WIP" - kernels=() + local pkg + local k_pkgs=() + local selected_k=() + local k_series=() # need to avoid most recent fedora kernel - if [ -x /usr/sbin/rpm ]; then + + if [ -x /usr/bin/rpm ]; then local kern_pkgs=( $( rpm -qa 'kernel*' | sort ) ) local pkg for pkg in "${kern_pkgs[@]}"; do - if [[ $pkg = kernel-tools* ]] \ - || [[ $pkg = kernel-headers* ]] \ - || [[ $pkg = kernel-devel* ]] ; then + if [[ $pkg = kernel-tools-* ]] \ + || [[ $pkg = kernel-headers-* ]] \ + || [[ $pkg = kernel-devel-* ]] ; then continue fi - kernels+=( $pkg ) - kernel_series=${pkg##kernel*5} - echo "K SER: $kernel_series" + k_pkgs+=( $pkg ) + done + for pkg in "${k_pkgs[@]}"; do + pkg=${pkg##kernel-modules-extra-} + pkg=${pkg##kernel-modules-} + pkg=${pkg##kernel-core-} + kernel_series=${pkg##kernel-} + #debug "K SER: $kernel_series" + if contains k_series $kernel_series; then + continue + else + k_series+=( $kernel_series ) + fi + done + IFS=$'\n' k_series=($(sort <<<"${k_series[*]}" | uniq)); unset IFS + for pkg in "${k_series[@]}"; do + debug "series $pkg" done fi - if (( $verbose > 0 )); then - printf "Would remove %s\n" "${kernels[@]}" + set +x + if (( ${#selected_k[@]} < 1 )); then + note "No kernels selected for removal" + fi + if (( $quiet < 1 )); then + printf "Would remove %s\n" "${selected_k[@]}" fi } @@ -346,16 +370,16 @@ survey_report_data() { # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # survey_areas() { local area - if [[ x$quiet = x ]] || (( $quiet < 1 )); then - echo -n "Surveying..." - fi + note "Surveying..." for area in "${!surveyors_map[@]}"; do - if [[ x$quiet = x ]] || (( $quiet < 1 )); then - echo -n "#" + if (( $quiet < 1 )) && (( $verbose < 1 )); then + note -n "#" fi ${surveyors_map[$area]} done - echo "" + if (( $quiet < 1 )) && (( $verbose < 1 )); then + echo "" + fi } # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # @@ -370,9 +394,12 @@ survey_areas disk_usage_report if (( ${#core_files[@]} > 0 )); then - note "Core Files detected, will remove:" + note "Core Files detected:" hr - printf ' %s\n' "${core_files[@]}" + #printf ' %s\n' "${core_files[@]}" + for core in "${core_files[@]}"; do + file $core + done hr selections+=("c") fi @@ -400,10 +427,8 @@ if contains "selections" "a" ; then debug "Will perform ${desc[$z]}" ${cleaners_map[$z]} done - survey_areas disk_usage_report - exit 0 fi @@ -414,7 +439,6 @@ if (( ${#selections[@]} > 0 )) ; then ${cleaners_map[$z]} selections=("${selections[@]/$z}") done - survey_areas disk_usage_report fi @@ -423,8 +447,6 @@ set +x # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # ask for things to remove if we are interactive # # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # - - choice="" while [[ $choice != q ]]; do hr