diff --git a/cicd/README.txt b/cicd/README.txt index 3015f1865..b0f58827e 100644 --- a/cicd/README.txt +++ b/cicd/README.txt @@ -9,15 +9,18 @@ Implementation: * Web server accessible to all CICD test beds runs a 'test orchestrator' logic, henceforth TO * This TO will periodically query jfrog for latest openwrt builds (see jfrog.pl) - * If new build is found, a text file containing pertinent info, including the HW platform + * If new build is found, a work-item file containing pertinent info, including the HW platform will be created, example: -CICD_URL=https://tip.jfrog.io/artifactory/tip-wlan-ap-firmware/ -CICD_FILE_NAME=ea8300-2020-04-24-046ab4f.tar.gz -CICD_URL_DATE=24-Apr-2020 18:28 +CICD_TYPE=fast +CICD_RPT_NAME=ea8300 +CICD_RPT_DIR=greearb@192.168.100.195:/var/www/html/tip/testbeds//ferndale-basic-01/reports CICD_HW=ea8300 -CICD_FILEDATE=2020-04-24 -CICD_GITHASH=046ab4f +CICD_FILEDATE= +CICD_GITHASH= +CICD_URL=https://tip.jfrog.io/artifactory/tip-wlan-ap-firmware/ +CICD_FILE_NAME=ea8300 +CICD_URL_DATE=24-Apr-2020 16:32 * TO has manually configured list of test-beds, with some info about each test bed, including the DUT HW platform and testing capabilities. @@ -26,6 +29,10 @@ CICD_GITHASH=046ab4f * The TO writes a new test configuration file into this directory. The test configuration file will have the info above, and also have other info including the tests to run and where to upload results when complete. + * TO looks for any completed results, and removes the work-item if result is found. + * TO will re-calculate historical charts and publish those if new results are found for a testbed. + It could generate email and/or poke the results into testrails or similar at this point. + * TO should run periodically every 1 minute or so to check on progress. * Test bed polling: @@ -34,15 +41,11 @@ CICD_GITHASH=046ab4f to the TO to indicate it is working on the test. * When test is complete, TB will upload results to TO server. TO now knows test bed is available for more jobs. - -* TO Polls periodically for results from test-beds, and when found it will re-generate historical - graphs and reports. If feasible, it could also email or otherwise notifiy whoever is interested in - these results. It could poke the results into testrails or similar at this point. + * TB should pause for 2 minutes after uploading results to make sure TO notices the new results and + removes the old work item so that TB does not re-test the same work item. -* If we can implement something like CTF, then it could cause the test config files to be placed into - the test-bed directory, potentially with URLs pointing to user-specified locations for testing private - builds. +* If we can implement something like CTF, it may take place of the Test Orchestrator. @@ -72,8 +75,10 @@ Run the jfrog.pl script from the tip/testbeds directory: A work-item file will be created as needed, in my case, it is here: -[greearb@ben-dt4 testbeds]$ cat ferndale-basic-01/pending_work/CICD_TEST-ea8300 -CICD_RPT=greearb@192.168.100.195:/var/www/html/tip/testbeds//ferndale-basic-01/reports/ea8300 +[greearb@ben-dt4 testbeds]$ cat ferndale-basic-01/pending_work/fast/CICD_TEST-ea8300 +CICD_TEST=fast +CICD_RPT_DIR=greearb@192.168.100.195:/var/www/html/tip/testbeds//ferndale-basic-01/reports/fast +CICD_RPT_NAME=ea8300 CICD_HW=ea8300 CICD_FILEDATE= CICD_GITHASH= @@ -86,6 +91,8 @@ CICD_URL_DATE=24-Apr-2020 16:32 ************ Installation / Usage on Test Controller ************** # Set up OS +sudo needs to work w/out password. + sudo chmod a+rwx /dev/ttyUSB* sudo pip3 install pexpect-serial diff --git a/cicd/jfrog.pl b/cicd/jfrog.pl index 2e6a6972c..923ad1d5d 100755 --- a/cicd/jfrog.pl +++ b/cicd/jfrog.pl @@ -3,6 +3,7 @@ # Query jfrog URL and get list of builds. # This will be run on the test-bed orchestrator # Run this in directory that contains the testbed_$hw/ directories +# Assumes cicd.class is found in ~/git/tip/wlan-lanforge-scripts/gui/ use strict; use warnings; @@ -15,6 +16,8 @@ my $files_processed = "jfrog_files_processed.txt"; my $tb_url_base = "cicd_user\@tip.cicd.cloud.com/testbeds"; # Used by SSH: scp -R results_dir cicd_user@tip.cicd.cloud.com/testbeds/ my $help = 0; my $cicd_prefix = "CICD_TEST"; +my $kpi_dir = "/home/greearb/git/tip/wlan-lanforge-scripts/gui/"; +my @ttypes = ("fast", "basic"); my $usage = qq($0 [--user { jfrog user (default: cicd_user) } @@ -51,14 +54,48 @@ if ($passwd eq "") { my $i; +my $pwd = `pwd`; +chomp($pwd); + +my $listing; +my @lines; +my $j; + +# Check for any completed reports. +for ($j = 0; $j<@ttypes; $j++) { + my $ttype = $ttypes[$j]; + $listing = `ls */reports/$ttype/NEW_RESULTS-*`; + @lines = split(/\n/, $listing); + for ($i = 0; $i<@lines; $i++) { + my $ln = $lines[$i]; + chomp($ln); + if ($ln =~ /(.*)\/NEW_RESULTS/) { + my $process = $1; + my $completed = `cat $ln`; + chomp($completed); + if ($ln =~ /(.*)\/reports\/$ttype\/NEW_RESULTS/) { + my $tbed = $1; + + `rm ./$tbed/pending_work/$completed`; + + my $cmd = "cd $kpi_dir && java kpi --dir \"$pwd/$process\" && cd -"; + print ("Running kpi: $cmd\n"); + `$cmd`; + `rm $ln`; + `scp -C -r $process www:candela_html/examples/cicd/$tbed/` + } + } + } +} + #Read in already_processed builds my @processed = (); -my $listing = `cat $files_processed`; -my @lines = split(/\n/, $listing); +$listing = `cat $files_processed`; +@lines = split(/\n/, $listing); for ($i = 0; $i<@lines; $i++) { my $ln = $lines[$i]; chomp($ln); - print("Skipping, already processed: $ln\n"); + print("Reported already processed: $ln\n"); push(@processed, $ln); } @@ -143,14 +180,33 @@ for ($i = 0; $i<@lines; $i++) { $fname_nogz = $1; } - my $work_fname = "$best_tb/pending_work/$cicd_prefix-$fname_nogz"; + my $ttype = "fast"; + my $work_fname = "$best_tb/pending_work/$cicd_prefix-$fname_nogz-$ttype"; open(FILE, ">", "$work_fname"); - system("mkdir -p $best_tb/reports"); + system("mkdir -p $best_tb/reports/$ttype"); - # In case we run different types of tests, report dir would need to be unique per test run - print FILE "CICD_RPT=$tb_url_base/$best_tb/reports/$fname_nogz\n"; + print FILE "CICD_TYPE=$ttype\n"; + print FILE "CICD_RPT_NAME=$fname_nogz\n"; + print FILE "CICD_RPT_DIR=$tb_url_base/$best_tb/reports/$ttype\n"; + + print FILE "CICD_HW=$hw\nCICD_FILEDATE=$fdate\nCICD_GITHASH=$githash\n"; + print FILE "CICD_URL=$url\nCICD_FILE_NAME=$fname\nCICD_URL_DATE=$date\n"; + + close(FILE); + + + $ttype = "basic"; + $work_fname = "$best_tb/pending_work/$cicd_prefix-$fname_nogz-$ttype"; + + open(FILE, ">", "$work_fname"); + + system("mkdir -p $best_tb/reports/$ttype"); + + print FILE "CICD_TYPE=$ttype\n"; + print FILE "CICD_RPT_NAME=$fname_nogz\n"; + print FILE "CICD_RPT_DIR=$tb_url_base/$best_tb/reports/$ttype\n"; print FILE "CICD_HW=$hw\nCICD_FILEDATE=$fdate\nCICD_GITHASH=$githash\n"; print FILE "CICD_URL=$url\nCICD_FILE_NAME=$fname\nCICD_URL_DATE=$date\n"; @@ -160,6 +216,11 @@ for ($i = 0; $i<@lines; $i++) { print("Next: File Name: $fname Display Name: $name Date: $date\n"); print("Work item placed at: $work_fname\n"); #print("To download: curl --location -o /tmp/$fname -u $user:$passwd $url/$fname\n"); + + # Note this one is processed + `echo -n "$fname " >> $files_processed`; + `date >> $files_processed`; + exit(0); } diff --git a/cicd/testbed_poll.pl b/cicd/testbed_poll.pl index 01bf68a07..843a508a3 100755 --- a/cicd/testbed_poll.pl +++ b/cicd/testbed_poll.pl @@ -71,11 +71,20 @@ for ($i = 0; $i<@lines; $i++) { my $ln = $lines[$i]; chomp($ln); - if ($ln =~ /href=\"(CICD_TEST-.*)\">(.*)<\/a>\s+(.*)\s+\S+\s+\S+/) { - my $fname = $1; - my $name = $2; - my $date = $3; + my $fname = ""; + my $name = ""; + my $date = ""; + if ($ln =~ /href=\"(CICD_TEST-.*)\">(.*)<\/a>\s+(.*)\s+\S+\s+\S+/) { + $fname = $1; + $name = $2; + $date = $3; + } + elsif ($ln =~ /href=\"(CICD_TEST-.*)\">(.*)<\/a>/) { + $fname = $1; + } + + if ($fname ne "") { # Grab that test file $cmd = "curl --location $cuser -o $next_info $url/$fname"; do_system($cmd); @@ -84,8 +93,10 @@ for ($i = 0; $i<@lines; $i++) { my $jurl = ""; my $jfile = ""; my $report_to = ""; + my $report_name = ""; my $swver = ""; my $fdate = ""; + my $ttype = ""; my $listing = do_system("cat $next_info"); my @lines = split(/\n/, $listing); for ($i = 0; $i<@lines; $i++) { @@ -94,12 +105,18 @@ for ($i = 0; $i<@lines; $i++) { if ($ln =~ /^CICD_URL=(.*)/) { $jurl = $1; } + elsif ($ln =~ /^CICD_TYPE=(.*)/) { + $ttype = $1; + } elsif ($ln =~ /^CICD_FILE_NAME=(.*)/) { $jfile = $1; } - elsif ($ln =~ /^CICD_RPT=(.*)/) { + elsif ($ln =~ /^CICD_RPT_DIR=(.*)/) { $report_to = $1; } + elsif ($ln =~ /^CICD_RPT_NAME=(.*)/) { + $report_name = $1; + } elsif ($ln =~ /^CICD_GITHASH=(.*)/) { $swver = $1; } @@ -108,11 +125,11 @@ for ($i = 0; $i<@lines; $i++) { } } - if ($swver = "") { + if ($swver eq "") { $swver = $fdate; } - if ($swver = "") { + if ($swver eq "") { $swver = "jfile"; } @@ -160,12 +177,23 @@ for ($i = 0; $i<@lines; $i++) { exit(1); } + my $gmport = "3990"; + my $gmanager = $lfmgr; + my $scenario = "tip-auto"; # matches basic_regression.bash + + if ($env =~ /GMANAGER=(.*)/g) { + $gmanager = $1; + } + if ($env =~ /GMPORT=(.*)/g) { + $gmport = $1; + } + # and then get it onto the DUT, reboot DUT, re-configure as needed, do_system("scp *sysupgrade.bin lanforge\@$lfmgr:tip-$jfile"); # TODO: Kill anything using the serial port - + do_system("sudo lsof -t $serial | sudo xargs --no-run-if-empty kill -9"); # and then kick off automated regression test. # Default gateway on the AP should be one of the ports on the LANforge system, so we can use @@ -177,11 +205,24 @@ for ($i = 0; $i<@lines; $i++) { } if ($ap_gw eq "") { print("ERROR: Could not find default gateway for AP, route info:\n$ap_route\n"); - # TODO: Re-apply scenario so the LANforge gateway/NAT is enabled for sure. + # Re-apply scenario so the LANforge gateway/NAT is enabled for sure. + do_system("../../lanforge/lanforge-scripts/lf_gui_cmd.pl --manager $gmanager --port $gmport --scenario $scenario"); # TODO: Use power-controller to reboot the AP and retry. - exit(1); + + my $out = do_system("../../lanforge/lanforge-scripts/openwrt_ctl.py --scheme serial --tty $serial --action reboot"); + print ("Reboot DUT to try to recover networking:\n$out\n"); + sleep(15); + + $ap_route = do_system("../../lanforge/lanforge-scripts/openwrt_ctl.py --scheme serial --tty $serial --action cmd --value \"ip route show\""); + if ($ap_route =~ /default via (\S+)/g) { + $ap_gw = $1; + } + if ($ap_gw eq "") { + exit(1); + } } + # TODO: Change this to curl download?? my $ap_out = do_system("../../lanforge/lanforge-scripts/openwrt_ctl.py --scheme serial --tty $serial --action sysupgrade --value \"lanforge\@$ap_gw:tip-$jfile\""); # System should be rebooted at this point. @@ -197,15 +238,22 @@ for ($i = 0; $i<@lines; $i++) { $ap_out = do_system("../../lanforge/lanforge-scripts/openwrt_ctl.py --scheme serial --tty $serial --action reboot"); print ("Reboot DUT so overlay takes effect:\n$ap_out\n"); - # TODO: Allow specifying other tests. - $ap_out = do_system("cd $tb_dir && DUT_SW_VER=$swver ./run_basic_fast.bash"); - print("Regression test script output:\n$ap_out\n"); + if ($ttype eq "fast") { + $ap_out = do_system("cd $tb_dir && DUT_SW_VER=$swver ./run_basic_fast.bash"); + } + else { + $ap_out = do_system("cd $tb_dir && DUT_SW_VER=$swver ./run_basic.bash"); + } + print("Regression $ttype test script output:\n$ap_out\n"); #When complete, upload the results to the requested location. if ($ap_out =~ /Results-Dir: (.*)/g) { my $rslts_dir = $1; print ("Found results at: $rslts_dir\n"); - do_system("scp -r $rslts_dir $report_to"); + do_system("mv $rslts_dir /tmp/$report_name"); + do_system("scp -r /tmp/$report_name $report_to/"); + do_system("echo $fname > /tmp/NEW_RESULTS-$fname"); + do_system("scp /tmp/NEW_RESULTS-$fname $report_to/"); } exit(0);