From 5b161d8619589cf4eeb76328b4dfd7f86d6c907b Mon Sep 17 00:00:00 2001 From: Logan Lipke Date: Mon, 28 Dec 2020 14:53:03 -0800 Subject: [PATCH 01/10] Fixed some bugs, testing WIP --- connectTest.py | 51 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/connectTest.py b/connectTest.py index e13c27fb..d504d8ef 100755 --- a/connectTest.py +++ b/connectTest.py @@ -15,7 +15,7 @@ from generic_cx import GenericCx mgrURL = "http://localhost:8080/" staName = "sta0" staNameUri = "port/1/1/" + staName - +suppress_related = True class ConnectTest(LFCliBase): def __init__(self, lfhost, lfport): @@ -47,7 +47,7 @@ class ConnectTest(LFCliBase): if response["interface"] is not None: print("removing old station") removePort(1, staName, mgrURL) - waitUntilPortsDisappear(1, mgrURL, [staName]) + waitUntilPortsDisappear(mgrURL, [staName]) time.sleep(1) url = "cli-json/add_sta" @@ -56,14 +56,15 @@ class ConnectTest(LFCliBase): "resource": 1, "radio": "wiphy0", "sta_name": staName, - "ssid": "jedway-wpa2-x2048-5-1", - "key": "jedway-wpa2-x2048-5-1", + "ssid": "jedway-wpa2-x2048-4-4", + "key": "jedway-wpa2-x2048-4-4", "mode": 0, "mac": "xx:xx:xx:xx:*:xx", "flags": (0x400 + 0x20000 + 0x1000000000) # create admin down } - super().json_post(url, data) - time.sleep(0.05) + super().json_post(url, data, suppress_related_commands_=suppress_related) + wait_until_ports_appear(mgrURL, [staName], True) + time.sleep(8) reqURL = "cli-json/set_port" data = { "shelf": 1, @@ -72,7 +73,7 @@ class ConnectTest(LFCliBase): "current_flags": (0x1 + 0x80000000), "interest": (0x2 + 0x4000 + 0x800000) # current, dhcp, down, } - super().json_post(reqURL, data) + super().json_post(reqURL, data, suppress_related_commands_=suppress_related) time.sleep(0.5) super().json_post("cli-json/set_port", portUpRequest(1, staName)) @@ -81,7 +82,7 @@ class ConnectTest(LFCliBase): "resource": 1, "port": staName, "probe_flags": 1} - super().json_post(reqURL, data) + super().json_post(reqURL, data, suppress_related_commands_=suppress_related) time.sleep(0.5) waitUntilPortsAdminUp(1, mgrURL, [staName]) @@ -121,11 +122,11 @@ class ConnectTest(LFCliBase): "url": "dl http://localhost/ /dev/null", "proxy_port" : "NA" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) data = { "endpoint": "all" } - super().json_post("/cli-json/nc_show_endpoints", data) + super().json_post("/cli-json/nc_show_endpoints", data, suppress_related_commands_=suppress_related) time.sleep(5) # create fileio endpoint @@ -138,7 +139,7 @@ class ConnectTest(LFCliBase): "type": "fe_nfs", "directory": "/mnt/fe-test" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) time.sleep(1) data = { "endpoint": "all" @@ -158,7 +159,7 @@ class ConnectTest(LFCliBase): data = { "endpoint": "all" } - super().json_post("/cli-json/nc_show_endpoints", data) + super().json_post("/cli-json/nc_show_endpoints", data, suppress_related_commands_=suppress_related) # create redirects for wanlink url = "/cli-json/add_rdd" @@ -168,7 +169,7 @@ class ConnectTest(LFCliBase): "port": "rdd0", "peer_ifname": "rdd1" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) url = "/cli-json/add_rdd" data = { @@ -177,7 +178,7 @@ class ConnectTest(LFCliBase): "port": "rdd1", "peer_ifname": "rdd0" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) time.sleep(.05) # reset redirect ports @@ -187,7 +188,7 @@ class ConnectTest(LFCliBase): "resource": 1, "port": "rdd0" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) url = "/cli-json/reset_port" data = { @@ -195,7 +196,7 @@ class ConnectTest(LFCliBase): "resource": 1, "port": "rdd1" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) time.sleep(.05) # create wanlink endpoints @@ -208,7 +209,7 @@ class ConnectTest(LFCliBase): "latency": 20, "max_rate": 1544000 } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) url = "/cli-json/add_wl_endp" data = { @@ -219,12 +220,12 @@ class ConnectTest(LFCliBase): "latency": 30, "max_rate": 1544000 } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) time.sleep(.05) data = { "endpoint": "all" } - super().json_post("/cli-json/nc_show_endpoints", data) + super().json_post("/cli-json/nc_show_endpoints", data, suppress_related_commands_=suppress_related) time.sleep(10) @@ -246,7 +247,7 @@ class ConnectTest(LFCliBase): "tx_endp": "l4Test", "rx_endp": "NA" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) time.sleep(.05) # create fileio cx @@ -257,7 +258,7 @@ class ConnectTest(LFCliBase): "tx_endp": "fioTest", "rx_endp": "NA" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) time.sleep(.05) # create generic cx @@ -268,7 +269,7 @@ class ConnectTest(LFCliBase): "tx_endp": "genTest1", "rx_endp": "genTest2" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) time.sleep(.05) # create wanlink cx @@ -279,12 +280,12 @@ class ConnectTest(LFCliBase): "tx_endp": "wlan0", "rx_endp": "wlan1" } - super().json_post(url, data) + super().json_post(url, data, suppress_related_commands_=suppress_related) time.sleep(.5) data = { "endpoint": "all" } - super().json_post("/cli-json/nc_show_endpoints", data) + super().json_post("/cli-json/nc_show_endpoints", data, suppress_related_commands_=suppress_related) cxNames = ["testTCP", "testUDP", "CX_l4Test", "CX_fioTest", "CX_genTest1", "CX_wlan0"] @@ -451,7 +452,7 @@ class ConnectTest(LFCliBase): "resource": 1, "port": staName } - super().json_post(reqURL, data) + super().json_post(reqURL, data, suppress_related_commands_=suppress_related) endpNames = ["testTCP-A", "testTCP-B", "testUDP-A", "testUDP-B", From 60b7c0d9dd0f391fd419b7cee69b5f7405adb9d2 Mon Sep 17 00:00:00 2001 From: Dipti Date: Mon, 28 Dec 2020 15:25:51 -0800 Subject: [PATCH 02/10] updated args and example --- py-scripts/test_all_scripts.sh | 4 +- py-scripts/test_ipv4_l4_urls_per_ten.py | 28 +++++++---- py-scripts/test_ipv4_ps.py | 23 ++++++--- py-scripts/test_ipv4_ttls.py | 64 +++++++++++++++---------- 4 files changed, 76 insertions(+), 43 deletions(-) diff --git a/py-scripts/test_all_scripts.sh b/py-scripts/test_all_scripts.sh index ad05f922..ad3b7fcb 100755 --- a/py-scripts/test_all_scripts.sh +++ b/py-scripts/test_all_scripts.sh @@ -3,7 +3,8 @@ # OPTION ONE: ./test_all_scripts.sh : this command runs all the scripts in the array "testCommands" # OPTION TWO: ./test_all_scripts.sh 4 5 : this command runs py-script commands (in testCommands array) that include the py-script options beginning with 4 and 5 (inclusive) in case function ret_case_num. #Variables -NUM_STA=4 +NUM_STA=5 +#ssid defaults to WPA2 security, ssid, and ssid key SSID_USED="jedway-wpa2-x2048-5-3" PASSWD_USED="jedway-wpa2-x2048-5-3" RADIO_USED="wiphy1" @@ -26,6 +27,7 @@ testCommands=("./example_security_connection.py --num_stations $NUM_STA --ssid j "./test_ipv4_l4_wifi.py --upstream_port eth1 --radio wiphy0 --num_stations $NUM_STA --security $SECURITY --ssid jedway-wpa2-x2048-4-4 --passwd jedway-wpa2-x2048-4-4 --test_duration 3m" "./test_ipv4_l4.py --radio wiphy3 --num_stations 4 --security wpa2 --ssid jedway-wpa2-x2048-4-1 --passwd jedway-wpa2-x2048-4-1 --url \"dl http://10.40.0.1 /dev/null\" --test_duration 2m --debug" "./test_ipv4_variable_time.py --radio wiphy1 --ssid jedway-wpa2-x2048-4-1 --passwd jedway-wpa2-x2048-4-1 --security wpa2 --mode 4 --ap 00:0e:8e:ff:86:e6 --test_duration 30s" + "./test_ipv4_l4_ftp_urls_per_ten.py --radio wiphy0 --ssid jedway-wpa2-x2048-4-1 --passwd jedway-wpa2-x2048-4-1 --security wpa2 --test_duration 30s --url \"ul ftp://lanforge:lanforge@10.40.0.1/example.txt /home/lanforge/example.txt\"" ) declare -A name_to_num name_to_num=( diff --git a/py-scripts/test_ipv4_l4_urls_per_ten.py b/py-scripts/test_ipv4_l4_urls_per_ten.py index 4d80510d..dbe5ec8f 100755 --- a/py-scripts/test_ipv4_l4_urls_per_ten.py +++ b/py-scripts/test_ipv4_l4_urls_per_ten.py @@ -175,15 +175,25 @@ python3 ./test_ipv4_l4_urls_per_ten.py --test_duration 2m --debug ''') - required = parser.add_argument_group('required arguments') - optional = parser.add_argument_group('optional arguments') - parser.add_argument('--requests_per_ten', help='--requests_per_ten number of request per ten minutes', default=600) - parser.add_argument('--num_tests', help='--num_tests number of tests to run. Each test runs 10 minutes', default=1) - parser.add_argument('--url', help='--url specifies upload/download, address, and dest',default="dl http://10.40.0.1 /dev/null") - parser.add_argument('--test_duration', help='duration of test',default="2m") - parser.add_argument('--target_per_ten', help='--target_per_ten target number of request per ten minutes. test will check for 90 percent this value',default=600) - optional.add_argument('--mode',help='Used to force mode of stations') - optional.add_argument('--ap',help='Used to force a connection to a particular AP') + required = None + for agroup in parser._action_groups: + if agroup.title == "required arguments": + required = agroup + #if required is not None: + + optional = None + for agroup in parser._action_groups: + if agroup.title == "optional arguments": + optional = agroup + + if optional is not None: + optional.add_argument('--requests_per_ten', help='--requests_per_ten number of request per ten minutes', default=600) + optional.add_argument('--num_tests', help='--num_tests number of tests to run. Each test runs 10 minutes', default=1) + optional.add_argument('--url', help='--url specifies upload/download, address, and dest',default="dl http://10.40.0.1 /dev/null") + optional.add_argument('--test_duration', help='duration of test',default="2m") + optional.add_argument('--target_per_ten', help='--target_per_ten target number of request per ten minutes. test will check for 90 percent this value',default=600) + optional.add_argument('--mode',help='Used to force mode of stations') + optional.add_argument('--ap',help='Used to force a connection to a particular AP') args = parser.parse_args() num_sta = 2 diff --git a/py-scripts/test_ipv4_ps.py b/py-scripts/test_ipv4_ps.py index a0faf759..0659be61 100755 --- a/py-scripts/test_ipv4_ps.py +++ b/py-scripts/test_ipv4_ps.py @@ -19,7 +19,8 @@ import datetime class IPV4VariableTime(LFCliBase): - def __init__(self, host, port, ssid, security, password, sta_list, name_prefix, upstream, radio, + def __init__(self, ssid, security, password, sta_list, name_prefix, upstream, radio, + host="localhost", port=8080, side_a_min_rate=56, side_a_max_rate=0, side_b_min_rate=56, side_b_max_rate=0, number_template="00000", test_duration="5m", use_ht160=False, @@ -169,7 +170,6 @@ class IPV4VariableTime(LFCliBase): def main(): - lfjson_port = 8080 parser = LFCliBase.create_basic_argparse( prog='test_ipv4_variable_time.py', @@ -192,11 +192,20 @@ Generic command layout: --test_duration 2m --debug ''') - required = parser.add_argument_group('required arguments') - required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True) - parser.add_argument('--a_min', help='--a_min bps rate minimum for side_a', default=256000) - parser.add_argument('--b_min', help='--b_min bps rate minimum for side_b', default=256000) - parser.add_argument('--test_duration', help='--test_duration sets the duration of the test', default="5m") + required = None + for agroup in parser._action_groups: + if agroup.title == "required arguments": + required = agroup + #if required is not None: + + optional = None + for agroup in parser._action_groups: + if agroup.title == "optional arguments": + optional = agroup + if optional is not None: + optional.add_argument('--a_min', help='--a_min bps rate minimum for side_a', default=256000) + optional.add_argument('--b_min', help='--b_min bps rate minimum for side_b', default=256000) + optional.add_argument('--test_duration', help='--test_duration sets the duration of the test', default="5m") args = parser.parse_args() num_sta = 2 diff --git a/py-scripts/test_ipv4_ttls.py b/py-scripts/test_ipv4_ttls.py index 62c70f2e..0985ed67 100755 --- a/py-scripts/test_ipv4_ttls.py +++ b/py-scripts/test_ipv4_ttls.py @@ -20,7 +20,7 @@ import pprint class TTLSTest(LFCliBase): - def __init__(self, host, port, + def __init__(self, host="localhost", port=8080, ssid="[BLANK]", security="wpa2", password="[BLANK]", @@ -203,10 +203,12 @@ class TTLSTest(LFCliBase): if (len(sta_list) == len(ip_map)) and (len(sta_list) == len(associated_map)): self._pass("PASS: All stations associated with IP", print_pass) else: + self._fail("FAIL: Not all stations able to associate/get IP", print_fail) - print("sta_list", sta_list) - print("ip_map", ip_map) - print("associated_map", associated_map) + if self.debug: + print("sta_list", sta_list) + print("ip_map", ip_map) + print("associated_map", associated_map) # notice that this does not actually generate traffic # please see test_ipv4_variable_time for example of generating traffic @@ -225,8 +227,6 @@ class TTLSTest(LFCliBase): debug=self.debug) def main(): - lfjson_host = "localhost" - lfjson_port = 8080 parser = LFCliBase.create_basic_argparse( prog='test_ipv4_ttls.py', @@ -238,31 +238,43 @@ def main(): test_ipv4_ttls.py: -------------------- Generic command layout: - python ./test_ipv4_ttls.py --upstream_port - --radio + python ./test_ipv4_ttls.py + + --upstream_port eth1 + --radio wiphy0 --num_stations 3 --ssid ssid-wpa-1 - --keyphrase ssid-wpa-1 - --security + --key ssid-wpa-1 + --security --debug ''') - required = parser.add_argument_group('required arguments') - required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True) - parser.add_argument('--a_min', help='--a_min bps rate minimum for side_a', default=256000) - parser.add_argument('--b_min', help='--b_min bps rate minimum for side_b', default=256000) - parser.add_argument('--test_duration', help='--test_duration sets the duration of the test', default="5m") - parser.add_argument('--key-mgmt', help="--key-mgt: { %s }"%", ".join(realm.wpa_ent_list()), default="WPA-EAP") - parser.add_argument('--wpa_psk', help='wpa-ent pre shared key', default="[BLANK]") - parser.add_argument('--eap', help='--eap eap method to use', default="TTLS") - parser.add_argument('--identity', help='--identity eap identity string', default="testuser") - parser.add_argument('--ttls_passwd', help='--ttls_passwd eap password string', default="testpasswd") - parser.add_argument('--ttls_realm', help='--ttls_realm 802.11u home realm to use', default="localhost.localdomain") - parser.add_argument('--domain', help='--domain 802.11 domain to use', default="localhost.localdomain") - parser.add_argument('--hessid', help='--hessid 802.11u HESSID (MAC addr format/peer for WDS)', default="00:00:00:00:00:01") - parser.add_argument('--ieee80211w', help='--ieee80211w 0): From ba5a2c5086d5e8edefcc8a865cf2267e4bd5dfb4 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Mon, 28 Dec 2020 12:05:15 -0800 Subject: [PATCH 03/10] check_large_files.bash: deletes ~lanforge/Downloads --- check_large_files.bash | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index 384ae49f..daa63e68 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -232,10 +232,13 @@ clean_core_files() { } clean_lf_downloads() { - note "Clean LF downloads WIP" + note "Clean LF downloads..." if (( $verbose > 0 )); then - printf "%s\n" "${lf_downloads[@]}" + printf "%s " < <(echo "${lf_downloads[@]}" | sort | uniq) fi + cd /home/lanforge/Downloads + echo "${lf_downloads[@]}" | xargs rm -f + cd - } clean_ath10_files() { @@ -302,12 +305,12 @@ survey_core_files() { } # downloads -downloads=() +lf_downloads=() survey_lf_downloads() { debug "Surveying /home/lanforge downloads" cd /home/lanforge/Downloads || return 1 - mapfile -t downloads < <(ls *gz *z2 *-Installer.exe *firmware* kinst_* *Docs* 2>/dev/null) - totals[d]=$(du -hc "${downloads[@]}" | awk '/total/{print $1}') + mapfile -t lf_downloads < <(ls *gz *z2 *-Installer.exe *firmware* kinst_* *Docs* 2>/dev/null) + totals[d]=$(du -hc "${lf_downloads[@]}" | awk '/total/{print $1}') [[ x${totals[d]} = x ]] && totals[d]=0 } From 97489135287a50b27d33e24c471b9883f6f86ec9 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Mon, 28 Dec 2020 22:03:19 -0800 Subject: [PATCH 04/10] updateTest.bash: fixes missing $, does not attempt to trap KILL, apparently not possible --- updateTest.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/updateTest.bash b/updateTest.bash index f1c03f4b..52dcee55 100755 --- a/updateTest.bash +++ b/updateTest.bash @@ -24,7 +24,7 @@ D_MODE="${HL}/LANforgeGUI_${verNum}/DAEMON_MODE" trap do_sigint HUP trap do_sigint ABRT trap do_sigint INT -trap do_sigint KILL +# trap do_sigint KILL # cannot be trapped trap do_sigint PIPE trap do_sigint QUIT trap do_sigint SEGV @@ -129,7 +129,7 @@ touch "$NO_AUTO" pgrep java &>/dev/null && killall -9 java touch $GUIUpdate touch $ST -if [ ! -z "SKIP_INSTALL" ] && [ x$SKIP_INSTALL = x1 ]; then +if [ ! -z "$SKIP_INSTALL" ] && [ x$SKIP_INSTALL = x1 ]; then echo "skipping installation" | tee -a $GUIUpdate else echo "doing installation" From 3d36585f0a1927db59d3710df552fcc373b5f74a Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Mon, 28 Dec 2020 22:05:05 -0800 Subject: [PATCH 05/10] check_large_files.bash: WIP core files --- check_large_files.bash | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index daa63e68..1ec5db12 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -225,9 +225,12 @@ clean_old_kernels() { } clean_core_files() { - note "Cleaning core files WIP" + note "Cleaning core files..." + if (( ${#core_files[@]} < 1 )); then + return 0 + fi if (( $verbose > 0 )); then - printf "%s\n" "${core_files[@]}" + printf " %s\n" < <( echo "${core_files[@]}" | sort | uniq ) fi } @@ -294,8 +297,8 @@ survey_core_files() { cd / #set -x mapfile -t core_files < <(ls /core* /home/lanforge/core* 2>/dev/null) 2>/dev/null - if [[ $verbose = 1 ]]; then - printf "%s\n" "${core_files[@]}" + if [[ $verbose = 1 ]] && (( ${#core_files[@]} > 0 )); then + printf " %s\n" "${core_files[@]}" | head fi if (( ${#core_files[@]} > 0 )); then totals[c]=$(du -hc "${core_files[@]}" | awk '/total/{print $1}') @@ -407,11 +410,26 @@ disk_usage_report if (( ${#core_files[@]} > 0 )); then note "Core Files detected:" hr - #printf ' %s\n' "${core_files[@]}" - for core in "${core_files[@]}"; do - file $core - done + filestr="" + declare -A core_groups + set -e + # note that the long pipe at the bottom of the loop is the best way to get + # the system to operate with thousands of core files + while read group7; do + (( $verbose > 0 )) && echo -n '.' + group7="${group7%, *}" + group7="#${group7//\'/}" + [[ ${core_groups[$group7]+_} != _ ]] && core_groups[$group7]=0 + core_groups[$group7]=$(( ${core_groups[$group7]} + 1 )) + + done < <(echo "${core_files[@]}" | xargs file | awk -F": " '/execfn:/{print $7}') + echo "" + echo "These types of core files were found:" + while read group; do + echo " $group -> ${core_groups[$group]}" + done < <(echo "${!core_groups[@]}" | sort ) hr + set +x selections+=("c") fi From 37c435d9bc5b89ca64c8f4210b7fcf26380c1dc7 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Tue, 29 Dec 2020 11:46:39 -0800 Subject: [PATCH 06/10] check_large_files.bash: cleans var log, cleans core files --- check_large_files.bash | 64 +++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index 1ec5db12..8c17da80 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -227,11 +227,19 @@ clean_old_kernels() { clean_core_files() { note "Cleaning core files..." if (( ${#core_files[@]} < 1 )); then + debug "No core files ?" return 0 fi - if (( $verbose > 0 )); then - printf " %s\n" < <( echo "${core_files[@]}" | sort | uniq ) - fi + local counter=0 + for f in "${core_files[@]}"; do + echo -n "-" + rm -f "$f" + counter=$(( counter + 1 )) + if (( ($counter % 100) == 0 )); then + sleep 0.2 + fi + done + echo "" } clean_lf_downloads() { @@ -252,10 +260,26 @@ clean_ath10_files() { } clean_var_log() { - note "Clean var log WIP" + note "Vacuuming journal..." + journalctl --vacuum-size 1M + if (( ${#var_log_files[@]} < 1 )); then + note "No notable files in /var/log to remove" + return + fi + local vee="" if (( $verbose > 0 )); then printf "%s\n" "${var_log_files[@]}" + vee="-v" fi + cd /var/log + while read file; do + if [[ $file = /var/log/messages ]]; then + echo "" > /var/log/messages + else + rm -f $vee "$file" + fi + done <<< "${var_log_files[@]}" + cd - } clean_mnt_fl_files() { @@ -330,7 +354,8 @@ survey_ath10_files() { var_log_files=() survey_var_log() { debug "Surveying var log" - mapfile -t var_log_files < <(find /var/log -type f -size +10M 2>/dev/null) + mapfile -t var_log_files < <(find /var/log -type f -size +35M \ + -not \( -path '*/journal/*' -o -path '*/sa/*' -o -path '*/lastlog' \) 2>/dev/null) totals[l]=$(du -hc "${var_log_files}" 2>/dev/null | awk '/total/{print $1}' ) [[ x${totals[l]} = x ]] && totals[l]=0 } @@ -387,7 +412,7 @@ survey_areas() { note "Surveying..." for area in "${!surveyors_map[@]}"; do if (( $quiet < 1 )) && (( $verbose < 1 )); then - note -n "#" + echo -n "#" fi ${surveyors_map[$area]} done @@ -408,29 +433,27 @@ survey_areas disk_usage_report if (( ${#core_files[@]} > 0 )); then - note "Core Files detected:" hr + note "${#core_files[@]} Core Files detected:" filestr="" declare -A core_groups - set -e + # set -e # note that the long pipe at the bottom of the loop is the best way to get # the system to operate with thousands of core files while read group7; do - (( $verbose > 0 )) && echo -n '.' + (( $verbose > 0 )) && echo -n '+' group7="${group7%, *}" - group7="#${group7//\'/}" + group7="${group7//\'/}" [[ ${core_groups[$group7]+_} != _ ]] && core_groups[$group7]=0 core_groups[$group7]=$(( ${core_groups[$group7]} + 1 )) - done < <(echo "${core_files[@]}" | xargs file | awk -F": " '/execfn:/{print $7}') echo "" echo "These types of core files were found:" - while read group; do - echo " $group -> ${core_groups[$group]}" - done < <(echo "${!core_groups[@]}" | sort ) + for group in "${!core_groups[@]}"; do + echo "${core_groups[$group]} files of $group" + done | sort -n hr - set +x - selections+=("c") + (( ${#core_files[@]} > 0 )) && selections+=("c") fi #echo "Usage of /mnt: $usage_mnt" @@ -462,16 +485,19 @@ if contains "selections" "a" ; then fi if (( ${#selections[@]} > 0 )) ; then - debug "Doing selected cleanup" + debug "Doing selected cleanup: " + printf " %s\n" "${selections[@]}" + sleep 1 for z in "${selections[@]}"; do - debug "Will perform ${desc[$z]}" + debug "Performing ${desc[$z]}" ${cleaners_map[$z]} selections=("${selections[@]/$z}") done survey_areas disk_usage_report +else + debug "No selections present" fi -set +x # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # ask for things to remove if we are interactive # From 0738dab2ad7d49b3129753ce902a36e2e3210722 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Tue, 29 Dec 2020 12:15:15 -0800 Subject: [PATCH 07/10] check_large_files.bash: cleans downloads, fixes current directory error --- check_large_files.bash | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index 8c17da80..aa1afe7c 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -10,6 +10,7 @@ deletion_targets=() show_menu=1 verbose=0 quiet=0 +starting_dir="$PWD" USAGE="$0 # Check for large files and purge many of the most inconsequencial -a # automatic: disable menu and clean automatically @@ -243,13 +244,22 @@ clean_core_files() { } clean_lf_downloads() { + if (( ${#lf_downloads[@]} < 1 )); then + note "No /home/lanforge/downloads files to remove" + return 0 + fi note "Clean LF downloads..." if (( $verbose > 0 )); then - printf "%s " < <(echo "${lf_downloads[@]}" | sort | uniq) + printf "Delete:[%s]\n" "${lf_downloads[@]}" | sort fi cd /home/lanforge/Downloads - echo "${lf_downloads[@]}" | xargs rm -f - cd - + for f in "${lf_downloads[@]}"; do + [[ "$f" = "/" ]] && echo "Whuuut? this is not good, bye." && exit 1 + echo "Next:[$f]" + sleep 0.2 + rm -f "$f" + done + cd "$starting_dir" } clean_ath10_files() { @@ -279,7 +289,7 @@ clean_var_log() { rm -f $vee "$file" fi done <<< "${var_log_files[@]}" - cd - + cd "$starting_dir" } clean_mnt_fl_files() { @@ -319,7 +329,6 @@ core_files=() survey_core_files() { debug "Surveying core files" cd / - #set -x mapfile -t core_files < <(ls /core* /home/lanforge/core* 2>/dev/null) 2>/dev/null if [[ $verbose = 1 ]] && (( ${#core_files[@]} > 0 )); then printf " %s\n" "${core_files[@]}" | head @@ -329,16 +338,19 @@ survey_core_files() { fi #set +x [[ x${totals[c]} = x ]] && totals[c]=0 + cd "$starting_dir" } # downloads lf_downloads=() survey_lf_downloads() { debug "Surveying /home/lanforge downloads" - cd /home/lanforge/Downloads || return 1 + [ ! -d "/home/lanforge/Downloads" ] && note "No downloads folder " && return 0 + cd /home/lanforge/Downloads mapfile -t lf_downloads < <(ls *gz *z2 *-Installer.exe *firmware* kinst_* *Docs* 2>/dev/null) totals[d]=$(du -hc "${lf_downloads[@]}" | awk '/total/{print $1}') [[ x${totals[d]} = x ]] && totals[d]=0 + cd "$starting_dir" } # Find ath10k crash residue @@ -363,12 +375,10 @@ survey_var_log() { # stuff in var tmp var_tmp_files=() survey_var_tmp() { - #set -x debug "Surveying var tmp" mapfile -t var_tmp_files < <(find /var/tmp -type f 2>/dev/null) totals[t]=$(du -sh "${var_tmp_files}" 2>/dev/null | awk '/total/{print $1}' ) [[ x${totals[t]} = x ]] && totals[t]=0 - #set +x } # Find size of /mnt/lf that is not mounted @@ -394,13 +404,12 @@ report_files=() survey_report_data() { debug "Surveying for lanforge report data" cd /home/lanforge - #set -x local fnum=$( find -type f -name '*.csv' 2>/dev/null | wc -l ) local fsiz=$( find -type f -name '*.csv' 2>/dev/null | xargs du -hc | awk '/total/{print $1}') totals[r]="$fsiz" [[ x${totals[r]} = x ]] && totals[r]=0 report_files=("CSV files: $fnum tt $fsiz") - #set +x + cd "$starting_dir" } @@ -470,7 +479,7 @@ fi # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # # delete extra things now # # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # -#set -x + if contains "selections" "a" ; then note "Automatic deletion will include: " printf "%s\n" "${selections[@]}" @@ -515,6 +524,7 @@ while [[ $choice != q ]]; do echo " m) orphaned /mnt/lf files ${totals[m]}" echo " r) compress .csv report files ${totals[r]}" echo " t) clean /var/tmp ${totals[t]}" + echo " q) quit" read -p "[1-5] or q ? " choice case "$choice" in From 8f5a72232bac1b9cc7e684632fb766c29e7add29 Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Tue, 29 Dec 2020 12:18:52 -0800 Subject: [PATCH 08/10] check_large_files.bash: slows down deletion on var tmp --- check_large_files.bash | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index aa1afe7c..bc100e05 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -292,7 +292,7 @@ clean_var_log() { cd "$starting_dir" } -clean_mnt_fl_files() { +clean_mnt_lf_files() { note "clean mnt lf files WIP" if (( $verbose > 0 )); then printf "%s\n" "${mnt_lf_files[@]}" @@ -310,9 +310,13 @@ compress_report_data() { clean_var_tmp() { note "clean var tmp" if (( $verbose > 0 )); then - printf "%s\n" "${var_tmp_files[@]}" + printf " %s\n" "${var_tmp_files[@]}" + sleep 1 fi - rf -f "${var_tmp_files[@]}" + for f in "${var_tmp_files[@]}"; do + rf -f "$f" + sleep 0.2 + done } From 69b5f27bf54c86fb6aa3f9d14e65bd4b8b35b02a Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Tue, 29 Dec 2020 12:39:32 -0800 Subject: [PATCH 09/10] check_large_files.bash: purge yum cache option --- check_large_files.bash | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/check_large_files.bash b/check_large_files.bash index bc100e05..d57df0e4 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -148,7 +148,8 @@ declare -A desc=( [k]="lf/ath10 files" [l]="/var/log" [m]="/mnt/lf files" - [r]="lf/report_data" + [n]="DNF cache" + [r]="/home/lanforge/report_data" [t]="/var/tmp" ) declare -A surveyors_map=( @@ -158,6 +159,7 @@ declare -A surveyors_map=( [k]="survey_ath10_files" [l]="survey_var_log" [m]="survey_mnt_lf_files" + [n]="survey_dnf_cache" [r]="survey_report_data" [t]="survey_var_tmp" ) @@ -169,6 +171,7 @@ declare -A cleaners_map=( [k]="clean_ath10_files" [l]="clean_var_log" [m]="clean_mnt_lf_files" + [n]="clean_dnf_cache" [r]="compress_report_data" [t]="clean_var_tmp" ) @@ -292,6 +295,14 @@ clean_var_log() { cd "$starting_dir" } +clean_dnf_cache() { + local yum="dnf" + which --skip-alias dnf &> /dev/null + (( $? < 0 )) && yum="yum" + debug "Purging $yum cache" + $yum clean all +} + clean_mnt_lf_files() { note "clean mnt lf files WIP" if (( $verbose > 0 )); then @@ -395,6 +406,14 @@ survey_mnt_lf_files() { [[ x${totals[m]} = x ]] && totals[m]=0 } +survey_dnf_cache() { + local yum="dnf" + which --skip-alias dnf &> /dev/null + (( $? < 0 )) && yum="yum" + debug "Surveying $yum cache" + totals[n]=$(du -hc '/var/cache/{dnf,yum}' 2>/dev/null | awk '/total/{print $1}') +} + ## Find size of /lib/modules # cd /lib/modules # mapfile -t usage_libmod < <(du -sh *) @@ -526,6 +545,7 @@ while [[ $choice != q ]]; do echo " k) ath10k crash files ${totals[k]}" echo " l) old /var/log files ${totals[l]}" echo " m) orphaned /mnt/lf files ${totals[m]}" + echo " n) purge dnf/yum cache ${totals[n]}" echo " r) compress .csv report files ${totals[r]}" echo " t) clean /var/tmp ${totals[t]}" echo " q) quit" From a2a3229df31644ad079d2b1af4de912658433bff Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Tue, 29 Dec 2020 14:24:37 -0800 Subject: [PATCH 10/10] check_large_files.bash: creates a way to sort kernels by expanded version number --- check_large_files.bash | 105 +++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 36 deletions(-) diff --git a/check_large_files.bash b/check_large_files.bash index d57df0e4..81286183 100755 --- a/check_large_files.bash +++ b/check_large_files.bash @@ -176,49 +176,82 @@ declare -A cleaners_map=( [t]="clean_var_tmp" ) +kernel_to_relnum() { + local hunks=() + #echo "KERNEL RELNUM:[$1]" + IFS="." read -ra hunks <<< "$1" + IFS= + local tmpstr + local max_width=8 + local last_len=0 + local diff_len=0 + local expandos=() + for i in 0 1 2; do + if (( $i < 2 )); then + expandos+=( $(( 100 + ${hunks[$i]} )) ) + else + tmpstr="0000000000000${hunks[i]}" + last_len=$(( ${#tmpstr} - $max_width )) + expandos+=( ${tmpstr:$last_len:${#tmpstr}} ) + #1>&2 echo "TRIMMED ${tmpstr:$last_len:${#tmpstr}}" + fi + done + #local relnum="${expandos[0]}${expandos[1]}${expandos[2]}" + echo "${expandos[0]}${expandos[1]}${expandos[2]}" +} + clean_old_kernels() { - note "Cleaning old kernels WIP" + note "Cleaning old kernels..." local pkg local k_pkgs=() local selected_k=() local k_series=() # need to avoid most recent fedora kernel - - if [ -x /usr/bin/rpm ]; then - local kern_pkgs=( $( rpm -qa 'kernel*' | sort -n ) ) - local pkg - for pkg in "${kern_pkgs[@]}"; do - if [[ $pkg = kernel-tools-* ]] \ - || [[ $pkg = kernel-headers-* ]] \ - || [[ $pkg = kernel-devel-* ]] ; then - continue - fi - k_pkgs+=( $pkg ) - done - for pkg in "${k_pkgs[@]}"; do - pkg=${pkg##kernel-modules-extra-} - pkg=${pkg##kernel-modules-} - pkg=${pkg##kernel-core-} - kernel_series=${pkg##kernel-} - #debug "K SER: $kernel_series" - if contains k_series $kernel_series; then - continue - else - k_series+=( $kernel_series ) - fi - done - IFS=$'\n' k_series=($(sort <<<"${k_series[*]}" | uniq)); unset IFS - for pkg in "${k_series[@]}"; do - debug "series $pkg" - done - if (( "${#k_series[@]}" > 1 )); then - local i=0 - # lets try and avoid the last item assuming that is the most recent - for i in $( seq 0 $(( ${#k_series[@]} - 2 )) ); do - debug "item $i is ${k_series[$i]}" - done - fi + if [ ! -x /usr/bin/rpm ]; then + note "Does not appear to be an rpm system." + return 0 fi + local ur=$( uname -r ) + local current_relnum=$( kernel_to_relnum $ur ) + local kern_pkgs=( $( rpm -qa 'kernel*' | sort ) ) + local pkg + for pkg in "${kern_pkgs[@]}"; do + if [[ $pkg = kernel-tools-* ]] \ + || [[ $pkg = kernel-headers-* ]] \ + || [[ $pkg = kernel-devel-* ]] ; then + continue + fi + k_pkgs+=( $pkg ) + done + for pkg in "${k_pkgs[@]}"; do + pkg=${pkg##kernel-modules-extra-} + pkg=${pkg##kernel-modules-} + pkg=${pkg##kernel-core-} + pkg=${pkg%.fc??.x86_64} + kernel_series=$( kernel_to_relnum ${pkg##kernel-} ) + + #debug "K SER: $kernel_series" + if contains k_series $kernel_series; then + continue + elif [[ x$current_relnum = x$kernel_series ]]; then + debug "avoiding current kernel [$kernel_series]" + else + k_series+=($kernel_series) + fi + done + + IFS=$'\n' k_series=($(sort <<<"${k_series[*]}" | uniq)); unset IFS + for pkg in "${k_series[@]}"; do + debug "series $pkg" + done + if (( "${#k_series[@]}" > 1 )); then + local i=0 + # lets try and avoid the last item assuming that is the most recent + for i in $( seq 0 $(( ${#k_series[@]} - 2 )) ); do + debug "item $i is ${k_series[$i]}" + done + fi + set +x if (( ${#selected_k[@]} < 1 )); then note "No kernels selected for removal"