cicd: Detect ath10k radio hard hang, attempt DUT reboot in that case.

And count this as if firmware crashed as far as log parsing is concerned.
This commit is contained in:
Ben Greear
2020-05-28 06:32:46 -07:00
parent 515151f80b
commit 3a807f19c1
2 changed files with 11 additions and 1 deletions

View File

@@ -110,6 +110,11 @@ function pre_test {
fi fi
} }
function reboot_dut {
../openwrt_ctl.py --action reboot --tty $AP_SERIAL --scheme serial
# TODO: Support hard-power cycle with power-ctl switch as well?
}
function post_test { function post_test {
DEST=$1 DEST=$1
mkdir -p $DEST/logs mkdir -p $DEST/logs
@@ -126,6 +131,9 @@ function post_test {
# Kill any existing processes on this serial port # Kill any existing processes on this serial port
pkill -f ".*openwrt_ctl.*$AP_SERIAL.*" pkill -f ".*openwrt_ctl.*$AP_SERIAL.*"
mv $MY_TMPDIR/dut_console_log.txt $DEST/logs/ mv $MY_TMPDIR/dut_console_log.txt $DEST/logs/
# detect a few fatal flaws and reqest AP restart if found.
grep "Hardware became unavailable" $DEST/logs/dut_console_log.txt && reboot_dut
fi fi
mv $MY_TMPDIR/basic_regression_log.txt $DEST/logs/test_automation_log.txt mv $MY_TMPDIR/basic_regression_log.txt $DEST/logs/test_automation_log.txt

View File

@@ -229,6 +229,7 @@ sub processLogs {
if (($ln =~ /WARNING:/) || if (($ln =~ /WARNING:/) ||
($ln =~ /BUG:/) || ($ln =~ /BUG:/) ||
($ln =~ /Hardware became unavailable during restart/) ||
($ln =~ /restarting hardware/) || ($ln =~ /restarting hardware/) ||
($ln =~ /crashed/)) { ($ln =~ /crashed/)) {
if ($ln =~ /WARNING:/) { if ($ln =~ /WARNING:/) {
@@ -240,7 +241,8 @@ sub processLogs {
elsif ($ln =~ /restarting hardware/) { elsif ($ln =~ /restarting hardware/) {
$restarting++; $restarting++;
} }
elsif ($ln =~ /crashed/) { elsif (($ln =~ /crashed/) || # software/firmware crashed
($ln =~ /became unavailable/)) { # hardware crashed
$crashed++; $crashed++;
} }
print IDX "<li><a href=$loghb#$tag>$enc_ln</a></li>\n"; print IDX "<li><a href=$loghb#$tag>$enc_ln</a></li>\n";