From 548167d7af4a816d2bbe0ebb7da1b2387ad59202 Mon Sep 17 00:00:00 2001 From: Ben Greear Date: Sat, 5 Feb 2022 20:56:29 -0800 Subject: [PATCH] Improve lf_ap_auto_test Some untested changes to regression test script to allow passing more info in on cmd line. Use correct DUT name when starting ap-auto test. Run connection test only at this time, it is fast when it works, and proves basic ability to launch the test. Signed-off-by: Ben Greear --- py-json/cv_dut_profile.py | 16 +++++----- py-json/cv_test_manager.py | 3 +- py-scripts/create_chamberview_dut.py | 23 ++++++++++++-- py-scripts/lf_ap_auto_test.py | 37 ++++++++++++++++------ py-scripts/regression_test.sh | 46 +++++++++++++--------------- 5 files changed, 78 insertions(+), 47 deletions(-) diff --git a/py-json/cv_dut_profile.py b/py-json/cv_dut_profile.py index 925e5a17..4df35d9e 100644 --- a/py-json/cv_dut_profile.py +++ b/py-json/cv_dut_profile.py @@ -6,7 +6,7 @@ import logging if sys.version_info[0] != 3: print("This script requires Python 3") - exit() + exit(1) sys.path.append(os.path.join(os.path.abspath(__file__ + "../../../"))) @@ -132,13 +132,13 @@ class cv_dut(LFCliBase): ssid_flags=0, ssid_flags_mask=0xFFFFFFFF): req_url = "/cli-json/add_dut_ssid" - logger.info("name: %s" % dut_name, - "ssid_idx: %s" % ssid_idx, - "ssid: %s" % ssid, - "passwd: %s" % passwd, - "bssid: %s" % bssid, - "ssid_flags: %s" % ssid_flags, - "ssid_flags_mask: %s" % ssid_flags_mask) + logger.info("DUT profile add-ssid: name: %s" % dut_name + + " ssid_idx: %s" % ssid_idx + + " ssid: %s" % ssid + + " passwd: %s" % passwd + + " bssid: %s" % bssid + + " ssid_flags: %s" % ssid_flags + + " ssid_flags_mask: %s" % ssid_flags_mask) self.json_post(req_url, { "name": dut_name, diff --git a/py-json/cv_test_manager.py b/py-json/cv_test_manager.py index 1203209a..88f84761 100644 --- a/py-json/cv_test_manager.py +++ b/py-json/cv_test_manager.py @@ -107,10 +107,9 @@ class cv_test(Realm): "text": text } - logger.info("adding- {text} to test config".format(text=text)) + logger.info("adding -:%s:- to test config: %s blob-name: %s" %(text, config_name, blob_test_name)) self.json_post(req_url, data) - # time.sleep(1) # Tell LANforge GUI Chamber View to launch a test def create_test(self, test_name, instance, load_old_cfg): diff --git a/py-scripts/create_chamberview_dut.py b/py-scripts/create_chamberview_dut.py index 3e31d852..474eee44 100755 --- a/py-scripts/create_chamberview_dut.py +++ b/py-scripts/create_chamberview_dut.py @@ -54,9 +54,12 @@ import importlib import argparse import time import shlex +import logging + +logger = logging.getLogger(__name__) if sys.version_info[0] != 3: - print("This script requires Python 3") + logger.critical("This script requires Python 3") exit(1) sys.path.append(os.path.join(os.path.abspath(__file__ + "../../../"))) @@ -67,7 +70,7 @@ dut = cv_dut_profile.cv_dut # from cv_test_manager import cv_test as cvtest cv_test_manager = importlib.import_module("py-json.cv_test_manager") cvtest = cv_test_manager.cv_test - +lf_logger_config = importlib.import_module("py-scripts.lf_logger_config") class DUT(dut): def __init__(self, @@ -134,7 +137,7 @@ class DUT(dut): #print("updated flag: %s" % (flag)) else: emsg = "ERROR: Un-supported security flag: %s" % (security) - print(emsg) + logger.critical(emsg) raise ValueError("Un-supported security flag") # Bad user input, terminate script. self.ssid[j]['flag'] = flag @@ -202,7 +205,21 @@ def main(): default=None, action='append') + # TODO: Use lfcli_base for common arguments. + parser.add_argument('--debug', help='Enable debugging', default=False, action="store_true") + parser.add_argument('--log_level', + default=None, + help='Set logging level: debug | info | warning | error | critical') + parser.add_argument('--lf_logger_config_json', + help="--lf_logger_config_json , json configuration of logger") + args = parser.parse_args() + + logger_config = lf_logger_config.lf_logger_config() + # set the logger level to requested value + logger_config.set_level(level=args.log_level) + logger_config.set_json(json_file=args.lf_logger_config_json) + new_dut = DUT(lfmgr=args.lfmgr, port=args.port, dut_name=args.dut_name, diff --git a/py-scripts/lf_ap_auto_test.py b/py-scripts/lf_ap_auto_test.py index be35357a..32bba670 100755 --- a/py-scripts/lf_ap_auto_test.py +++ b/py-scripts/lf_ap_auto_test.py @@ -167,9 +167,12 @@ import os import importlib import argparse import time +import logging + +logger = logging.getLogger(__name__) if sys.version_info[0] != 3: - print("This script requires Python 3") + logger.critical("This script requires Python 3") exit(1) sys.path.append(os.path.join(os.path.abspath(__file__ + "../../../"))) @@ -179,7 +182,7 @@ cvtest = cv_test_manager.cv_test cv_add_base_parser = cv_test_manager.cv_add_base_parser cv_base_adjust_parser = cv_test_manager.cv_base_adjust_parser LFUtils = importlib.import_module("py-json.LANforge.LFUtils") - +lf_logger_config = importlib.import_module("py-scripts.lf_logger_config") class ApAutoTest(cvtest): def __init__(self, @@ -208,9 +211,9 @@ class ApAutoTest(cvtest): raw_lines_file="", sets=None, graph_groups=None, - test_tag="" + debug=False, ): - super().__init__(lfclient_host=lf_host, lfclient_port=lf_port) + super().__init__(lfclient_host=lf_host, lfclient_port=lf_port, debug_=debug) if radio2 is None: radio2 = [] @@ -250,7 +253,6 @@ class ApAutoTest(cvtest): self.graph_groups = graph_groups self.lf_report_dir = lf_report_dir self.local_lf_report_dir = local_lf_report_dir - self.test_tag = test_tag def setup(self): # Nothing to do at this time. @@ -283,7 +285,7 @@ class ApAutoTest(cvtest): # Command line args take precedence. if self.upstream: - cfg_options.append("upstream-port: %s" % self.upstream) + cfg_options.append("upstream_port: %s" % self.upstream) if self.dut5_0 != "": cfg_options.append("dut5-0: " + self.dut5_0) if self.dut2_0 != "": @@ -294,8 +296,6 @@ class ApAutoTest(cvtest): cfg_options.append("max_stations_5: " + str(self.max_stations_5)) if self.max_stations_dual != -1: cfg_options.append("max_stations_dual: " + str(self.max_stations_dual)) - if self.test_tag != "": - cfg_options.append("test_tag: " + self.test_tag) # We deleted the scenario earlier, now re-build new one line at a time. self.build_cfg(self.config_name, blob_test, cfg_options) @@ -364,8 +364,22 @@ def main(): help="--lf_report_dir default '' put where dataplane script run from", default="") + # TODO: Use lfcli_base for common arguments. + parser.add_argument('--debug', help='Enable debugging', default=False, action="store_true") + parser.add_argument('--log_level', + default=None, + help='Set logging level: debug | info | warning | error | critical') + parser.add_argument('--lf_logger_config_json', + help="--lf_logger_config_json , json configuration of logger") + args = parser.parse_args() + logger_config = lf_logger_config.lf_logger_config() + # set the logger level to requested value + logger_config.set_level(level=args.log_level) + logger_config.set_json(json_file=args.lf_logger_config_json) + + cv_base_adjust_parser(args) CV_Test = ApAutoTest(lf_host=args.mgr, @@ -390,13 +404,18 @@ def main(): disables=args.disable, raw_lines=args.raw_line, raw_lines_file=args.raw_lines_file, - sets=args.set + sets=args.set, + debug=args.debug ) CV_Test.setup() CV_Test.run() CV_Test.check_influx_kpi(args) + if CV_Test.passes(): + CV_Test.exit_success() + else: + CV_Test.exit_fail() if __name__ == "__main__": main() diff --git a/py-scripts/regression_test.sh b/py-scripts/regression_test.sh index ad7d6417..cc0b3140 100755 --- a/py-scripts/regression_test.sh +++ b/py-scripts/regression_test.sh @@ -16,9 +16,13 @@ Help() echo "F is used to pass in an RC file which can store the credentials for running regression multiple times on your system" echo "H is used to test the help feature of each script, to make sure it renders properly." echo "L is used to give the IP address of the LANforge device which is under test" + echo "D is DUT5 string, used in ap-auto and similar chamber-view tests." + echo "2 is DUT2 string, used in ap-auto and similar chamber-view tests." + echo "r is 5Ghz radio, default is 1.1.wiphy1" + echo "M is 2.4Ghz radio, default is 1.1.wiphy0" echo "Example command: ./regression_test.sh -s SSID -p PASSWD -w SECURITY -m MGR" echo "Example command: ./regression_test.sh -s j-wpa2-153 -p j-wpa2-153 -w wpa2 -r 1.1.wiphy0 \\\\" - echo " e-M 1.1.wiphy1 -B 04:F0:21:CB:01:8B -V heather_ssid_2022 -m heather -R /tmp/ 1" + echo " -M 1.1.wiphy1 -B 04:F0:21:CB:01:8B -V heather_ssid_2022 -m heather -R /tmp/ 1" echo "If using the help flag, put the H flag at the end of the command after other flags." } @@ -27,8 +31,13 @@ REPORT_DIR="${HOMEPATH}/html-reports" TESTBED=UNKNOWN NOW=$(date +"%Y-%m-%d-%H-%M") NOW="${NOW/:/-}" +DUT2= +DUT5= +RADIO_USED="1.1.wiphy1" +RADIO5=$RADIO_USED +RADIO2="1.1.wiphy0" -while getopts ":h:s:S:p:w:m:r:R:F:B:u:U:D:H:M:C:e:V:E:T:" option; do +while getopts ":h:s:S:p:w:m:r:R:F:B:u:U:D:2:H:M:C:e:V:E:T:" option; do case "${option}" in h) # display Help Help @@ -54,6 +63,10 @@ while getopts ":h:s:S:p:w:m:r:R:F:B:u:U:D:H:M:C:e:V:E:T:" option; do ;; r) RADIO_USED=${OPTARG} + RADIO5=${OPTARG} + ;; + M) + RADIO2=${OPTARG} ;; R) REPORT_DIR=${OPTARG} @@ -74,14 +87,13 @@ while getopts ":h:s:S:p:w:m:r:R:F:B:u:U:D:H:M:C:e:V:E:T:" option; do ;; D) DUT5=${OPTARG} + ;; + 2) DUT2=${OPTARG} ;; H) ./lf_help_check.bash ;; - M) - RADIO2=${OPTARG} - ;; C) RESOURCE=${OPTARG} ;; @@ -150,14 +162,6 @@ if [[ ${#VAP_SSID} -eq 0 ]]; then VAP_SSID=SSID_USED fi -if [[ ${#RADIO_USED} -eq 0 ]]; then # Allow the user to change the radio they test against - RADIO_USED="1.1.wiphy1" -fi - -if [[ ${#RADIO2} -eq 0 ]]; then # Allow the user to change the radio they test against - RADIO2="1.1.wiphy0" -fi - if [[ ${#UPSTREAM_BARE} -eq 0 ]]; then UPSTREAM_BARE="eth1" fi @@ -191,10 +195,6 @@ NUM_STA=${NUM_STA:-4} TEST_HTTP_IP=${TEST_HTTP_IP:-10.40.0.1} COL_NAMES="name,tx_bytes,rx_bytes,dropped" -if [[ ${#DUT2} -eq 0 ]]; then - DUT5="linksys-8450 j-wpa2-153 c4:41:1e:f5:3f:25 (1)" - DUT2="linksys-8450 j-wpa2-153 c4:41:1e:f5:3f:25 (1)" -fi #CURR_TEST_NUM=0 CURR_TEST_NAME="BLANK" @@ -238,11 +238,6 @@ function create_station_and_dataplane() { set +x } function create_dut_and_chamberview() { - ./create_chamberview_dut.py --lfmgr $MGR --dut_name regression_dut \ - --ssid "ssid_idx=0 ssid='$SSID_USED' security='$SECURITY' password='$PASSWD_USED' bssid=$BSSID" - ./create_chamberview.py -m $MGR -cs 'regression_test' --delete_scenario \ - --line "Resource=1.$RESOURCE Profile=STA-AC Amount=1 Uses-1=$RADIO_USED Freq=-1 DUT=regression_dut DUT_Radio=$RADIO_USED Traffic=http" \ - --line "Resource=1.$RESOURCE Profile=upstream Amount=1 Uses-1=$UPSTREAM_BARE Uses-2=AUTO Freq=-1 DUT=regression_dut DUT_Radio=LAN Traffic=http" } function create_station_and_sensitivity { @@ -290,7 +285,7 @@ else testCommands=( "./create_bond.py --network_dev_list $RESOURCE.eth2,$UPSTREAM --bond_name $RESOURCE.bond5 --debug --mgr $MGR" "./create_bridge.py --target_device $RESOURCE.eth2,$UPSTREAM --bridge_name $RESOURCE.br5 --debug --mgr $MGR" - create_dut_and_chamberview + "./create_chamberview_dut.py --lfmgr $MGR --dut_name regression_dut --ssid \"ssid_idx=0 ssid='$SSID_USED' security='$SECURITY' password='$PASSWD_USED' bssid=$BSSID\" && ./create_chamberview.py -m $MGR -cs 'regression_test' --delete_scenario --line \"Resource=1.$RESOURCE Profile=STA-AC Amount=1 Uses-1=$RADIO_USED Freq=-1 DUT=regression_dut DUT_Radio=$RADIO_USED Traffic=http\" --line \"Resource=1.$RESOURCE Profile=upstream Amount=1 Uses-1=$UPSTREAM_BARE Uses-2=AUTO Freq=-1 DUT=regression_dut DUT_Radio=LAN Traffic=http\"" "./create_l3.py --radio $RADIO_USED --ssid $SSID_USED --password $PASSWD_USED --security $SECURITY --debug --mgr $MGR --endp_a wiphy0 --endp_b wiphy1" "./create_l3_stations.py --mgr $MGR --radio $RADIO_USED --ssid $SSID_USED --password $PASSWD_USED --security $SECURITY --debug" "./create_l4.py --radio $RADIO_USED --ssid $SSID_USED --password $PASSWD_USED --security $SECURITY --debug --mgr $MGR" @@ -319,12 +314,13 @@ else --max_stations_2 64 \ --max_stations_5 64 \ --max_stations_dual 64 \ - --radio2 $RADIO_USED \ + --radio5 $RADIO5 \ --radio2 $RADIO2 \ --set 'Basic Client Connectivity' 1 \ - --set 'Multi Band Performance' 1 \ + --set 'Multi Band Performance' 0 \ --set 'Skip 2.4Ghz Tests' 1 \ --set 'Skip 5Ghz Tests' 1 \ + --set 'Skip Dual-Band Tests' 1 \ --set 'Throughput vs Pkt Size' 0 \ --set 'Capacity' 0 \ --set 'Stability' 0 \