cicd: Support basic and fast test types.

End-to-end automated testing is mostly working, but still needs a bit of cleanup
and debugging.

cicd/README.txt

@@ -9,15 +9,18 @@ Implementation:
 
 * Web server accessible to all CICD test beds runs a 'test orchestrator' logic, henceforth TO
   *  This TO will periodically query jfrog for latest openwrt builds (see jfrog.pl)
-  *  If new build is found, a text file containing pertinent info, including the HW platform
+  *  If new build is found, a work-item file containing pertinent info, including the HW platform
      will be created, example:
 
-CICD_URL=https://tip.jfrog.io/artifactory/tip-wlan-ap-firmware/
+CICD_TYPE=fast
-CICD_FILE_NAME=ea8300-2020-04-24-046ab4f.tar.gz
+CICD_RPT_NAME=ea8300
-CICD_URL_DATE=24-Apr-2020 18:28 
+CICD_RPT_DIR=greearb@192.168.100.195:/var/www/html/tip/testbeds//ferndale-basic-01/reports
 CICD_HW=ea8300
-CICD_FILEDATE=2020-04-24
+CICD_FILEDATE=
-CICD_GITHASH=046ab4f
+CICD_GITHASH=
+CICD_URL=https://tip.jfrog.io/artifactory/tip-wlan-ap-firmware/
+CICD_FILE_NAME=ea8300
+CICD_URL_DATE=24-Apr-2020 16:32 
 
   *  TO has manually configured list of test-beds, with some info about each test
      bed, including the DUT HW platform and testing capabilities.
@@ -26,6 +29,10 @@ CICD_GITHASH=046ab4f
   *  The TO writes a new test configuration file into this directory.
      The test configuration file will have the info above, and also have other
      info including the tests to run and where to upload results when complete.
+  *  TO looks for any completed results, and removes the work-item if result is found.
+  *  TO will re-calculate historical charts and publish those if new results are found for a testbed.
+     It could generate email and/or poke the results into testrails or similar at this point.
+  *  TO should run periodically every 1 minute or so to check on progress.
 
 
 * Test bed polling:
@@ -34,15 +41,11 @@ CICD_GITHASH=046ab4f
      to the TO to indicate it is working on the test.
   *  When test is complete, TB will upload results to TO server.  TO now knows test bed is available
      for more jobs.
-
+  *  TB should pause for 2 minutes after uploading results to make sure TO notices the new results and
-* TO Polls periodically for results from test-beds, and when found it will re-generate historical
+     removes the old work item so that TB does not re-test the same work item.
-  graphs and reports.  If feasible, it could also email or otherwise notifiy whoever is interested in
-  these results.  It could poke the results into testrails or similar at this point.
 
 
-* If we can implement something like CTF, then it could cause the test config files to be placed into
+* If we can implement something like CTF, it may take place of the Test Orchestrator.
-  the test-bed directory, potentially with URLs pointing to user-specified locations for testing private
-  builds.
 
 
 
@@ -72,8 +75,10 @@ Run the jfrog.pl script from the tip/testbeds directory:
 
 A work-item file will be created as needed, in my case, it is here:
 
-[greearb@ben-dt4 testbeds]$ cat ferndale-basic-01/pending_work/CICD_TEST-ea8300
+[greearb@ben-dt4 testbeds]$ cat ferndale-basic-01/pending_work/fast/CICD_TEST-ea8300
-CICD_RPT=greearb@192.168.100.195:/var/www/html/tip/testbeds//ferndale-basic-01/reports/ea8300
+CICD_TEST=fast
+CICD_RPT_DIR=greearb@192.168.100.195:/var/www/html/tip/testbeds//ferndale-basic-01/reports/fast
+CICD_RPT_NAME=ea8300
 CICD_HW=ea8300
 CICD_FILEDATE=
 CICD_GITHASH=
@@ -86,6 +91,8 @@ CICD_URL_DATE=24-Apr-2020 16:32
 ************  Installation / Usage on Test Controller **************
 
 # Set up OS
+sudo needs to work w/out password.
+
 sudo chmod a+rwx /dev/ttyUSB*
 sudo pip3 install pexpect-serial
 

cicd/jfrog.pl

@@ -3,6 +3,7 @@
 # Query jfrog URL and get list of builds.
 # This will be run on the test-bed orchestrator
 # Run this in directory that contains the testbed_$hw/ directories
+# Assumes cicd.class is found in ~/git/tip/wlan-lanforge-scripts/gui/
 
 use strict;
 use warnings;
@@ -15,6 +16,8 @@ my $files_processed = "jfrog_files_processed.txt";
 my $tb_url_base = "cicd_user\@tip.cicd.cloud.com/testbeds";  # Used by SSH: scp -R results_dir cicd_user@tip.cicd.cloud.com/testbeds/
 my $help = 0;
 my $cicd_prefix = "CICD_TEST";
+my $kpi_dir = "/home/greearb/git/tip/wlan-lanforge-scripts/gui/";
+my @ttypes = ("fast", "basic");
 
 my $usage = qq($0
   [--user { jfrog user (default: cicd_user) }
@@ -51,14 +54,48 @@ if ($passwd eq "") {
 
 my $i;
 
+my $pwd = `pwd`;
+chomp($pwd);
+
+my $listing;
+my @lines;
+my $j;
+
+# Check for any completed reports.
+for ($j = 0; $j<@ttypes; $j++) {
+   my $ttype = $ttypes[$j];
+   $listing = `ls */reports/$ttype/NEW_RESULTS-*`;
+   @lines = split(/\n/, $listing);
+   for ($i = 0; $i<@lines; $i++) {
+      my $ln = $lines[$i];
+      chomp($ln);
+      if ($ln =~ /(.*)\/NEW_RESULTS/) {
+         my $process = $1;
+         my $completed = `cat $ln`;
+         chomp($completed);
+         if ($ln =~ /(.*)\/reports\/$ttype\/NEW_RESULTS/) {
+            my $tbed = $1;
+
+            `rm ./$tbed/pending_work/$completed`;
+
+            my $cmd = "cd $kpi_dir && java kpi --dir \"$pwd/$process\" && cd -";
+            print ("Running kpi: $cmd\n");
+            `$cmd`;
+            `rm $ln`;
+            `scp -C -r $process www:candela_html/examples/cicd/$tbed/`
+         }
+      }
+   }
+}
+
 #Read in already_processed builds
 my @processed = ();
-my $listing = `cat $files_processed`;
+$listing = `cat $files_processed`;
-my @lines = split(/\n/, $listing);
+@lines = split(/\n/, $listing);
 for ($i = 0; $i<@lines; $i++) {
    my $ln = $lines[$i];
    chomp($ln);
-   print("Skipping, already processed: $ln\n");
+   print("Reported already processed: $ln\n");
    push(@processed, $ln);
 }
 
@@ -143,14 +180,33 @@ for ($i = 0; $i<@lines; $i++) {
          $fname_nogz = $1;
       }
 
-      my $work_fname = "$best_tb/pending_work/$cicd_prefix-$fname_nogz";
+      my $ttype = "fast";
+      my $work_fname = "$best_tb/pending_work/$cicd_prefix-$fname_nogz-$ttype";
 
       open(FILE, ">", "$work_fname");
 
-      system("mkdir -p $best_tb/reports");
+      system("mkdir -p $best_tb/reports/$ttype");
 
-      # In case we run different types of tests, report dir would need to be unique per test run
+      print FILE "CICD_TYPE=$ttype\n";
-      print FILE "CICD_RPT=$tb_url_base/$best_tb/reports/$fname_nogz\n";
+      print FILE "CICD_RPT_NAME=$fname_nogz\n";
+      print FILE "CICD_RPT_DIR=$tb_url_base/$best_tb/reports/$ttype\n";
+
+      print FILE "CICD_HW=$hw\nCICD_FILEDATE=$fdate\nCICD_GITHASH=$githash\n";
+      print FILE "CICD_URL=$url\nCICD_FILE_NAME=$fname\nCICD_URL_DATE=$date\n";
+
+      close(FILE);
+
+
+      $ttype = "basic";
+      $work_fname = "$best_tb/pending_work/$cicd_prefix-$fname_nogz-$ttype";
+
+      open(FILE, ">", "$work_fname");
+
+      system("mkdir -p $best_tb/reports/$ttype");
+
+      print FILE "CICD_TYPE=$ttype\n";
+      print FILE "CICD_RPT_NAME=$fname_nogz\n";
+      print FILE "CICD_RPT_DIR=$tb_url_base/$best_tb/reports/$ttype\n";
 
       print FILE "CICD_HW=$hw\nCICD_FILEDATE=$fdate\nCICD_GITHASH=$githash\n";
       print FILE "CICD_URL=$url\nCICD_FILE_NAME=$fname\nCICD_URL_DATE=$date\n";
@@ -160,6 +216,11 @@ for ($i = 0; $i<@lines; $i++) {
       print("Next: File Name: $fname  Display Name: $name  Date: $date\n");
       print("Work item placed at: $work_fname\n");
       #print("To download: curl --location -o /tmp/$fname -u $user:$passwd $url/$fname\n");
+
+      # Note this one is processed
+      `echo -n "$fname " >> $files_processed`;
+      `date >> $files_processed`;
+
       exit(0);
    }
 

cicd/testbed_poll.pl

@@ -71,11 +71,20 @@ for ($i = 0; $i<@lines; $i++) {
    my $ln = $lines[$i];
    chomp($ln);
 
-   if ($ln =~ /href=\"(CICD_TEST-.*)\">(.*)<\/a>\s+(.*)\s+\S+\s+\S+/) {
+   my $fname = "";
-      my $fname = $1;
+   my $name = "";
-      my $name = $2;
+   my $date = "";
-      my $date = $3;
 
+   if ($ln =~ /href=\"(CICD_TEST-.*)\">(.*)<\/a>\s+(.*)\s+\S+\s+\S+/) {
+      $fname = $1;
+      $name = $2;
+      $date = $3;
+   }
+   elsif ($ln =~ /href=\"(CICD_TEST-.*)\">(.*)<\/a>/) {
+      $fname = $1;
+   }
+
+   if ($fname ne "") {
       # Grab that test file
       $cmd = "curl --location $cuser -o $next_info $url/$fname";
       do_system($cmd);
@@ -84,8 +93,10 @@ for ($i = 0; $i<@lines; $i++) {
       my $jurl = "";
       my $jfile = "";
       my $report_to = "";
+      my $report_name = "";
       my $swver = "";
       my $fdate = "";
+      my $ttype = "";
       my $listing = do_system("cat $next_info");
       my @lines = split(/\n/, $listing);
       for ($i = 0; $i<@lines; $i++) {
@@ -94,12 +105,18 @@ for ($i = 0; $i<@lines; $i++) {
          if ($ln =~ /^CICD_URL=(.*)/) {
             $jurl = $1;
          }
+         elsif ($ln =~ /^CICD_TYPE=(.*)/) {
+            $ttype = $1;
+         }
          elsif ($ln =~ /^CICD_FILE_NAME=(.*)/) {
             $jfile = $1;
          }
-         elsif ($ln =~ /^CICD_RPT=(.*)/) {
+         elsif ($ln =~ /^CICD_RPT_DIR=(.*)/) {
             $report_to = $1;
          }
+         elsif ($ln =~ /^CICD_RPT_NAME=(.*)/) {
+            $report_name = $1;
+         }
          elsif ($ln =~ /^CICD_GITHASH=(.*)/) {
             $swver = $1;
          }
@@ -108,11 +125,11 @@ for ($i = 0; $i<@lines; $i++) {
          }
       }
 
-      if ($swver = "") {
+      if ($swver eq "") {
          $swver = $fdate;
       }
 
-      if ($swver = "") {
+      if ($swver eq "") {
          $swver = "jfile";
       }
 
@@ -160,12 +177,23 @@ for ($i = 0; $i<@lines; $i++) {
          exit(1);
       }
 
+      my $gmport = "3990";
+      my $gmanager = $lfmgr;
+      my $scenario = "tip-auto";  # matches basic_regression.bash
+
+      if ($env =~ /GMANAGER=(.*)/g) {
+         $gmanager = $1;
+      }
+      if ($env =~ /GMPORT=(.*)/g) {
+         $gmport = $1;
+      }
+
       # and then get it onto the DUT, reboot DUT, re-configure as needed,
       do_system("scp *sysupgrade.bin lanforge\@$lfmgr:tip-$jfile");
 
 
       # TODO:  Kill anything using the serial port
-
+      do_system("sudo lsof -t $serial | sudo xargs --no-run-if-empty kill -9");
 
       # and then kick off automated regression test.
       # Default gateway on the AP should be one of the ports on the LANforge system, so we can use
@@ -177,11 +205,24 @@ for ($i = 0; $i<@lines; $i++) {
       }
       if ($ap_gw eq "") {
          print("ERROR:  Could not find default gateway for AP, route info:\n$ap_route\n");
-         # TODO:  Re-apply scenario so the LANforge gateway/NAT is enabled for sure.
+         # Re-apply scenario so the LANforge gateway/NAT is enabled for sure.
+         do_system("../../lanforge/lanforge-scripts/lf_gui_cmd.pl --manager $gmanager --port $gmport --scenario $scenario");
          # TODO:  Use power-controller to reboot the AP and retry.
+
+         my $out = do_system("../../lanforge/lanforge-scripts/openwrt_ctl.py --scheme serial --tty $serial --action reboot");
+         print ("Reboot DUT to try to recover networking:\n$out\n");
+         sleep(15);
+
+         $ap_route = do_system("../../lanforge/lanforge-scripts/openwrt_ctl.py --scheme serial --tty $serial --action cmd --value \"ip route show\"");
+         if ($ap_route =~ /default via (\S+)/g) {
+            $ap_gw = $1;
+         }
+         if ($ap_gw eq "") {
             exit(1);
          }
+      }
 
+      # TODO: Change this to curl download??
       my $ap_out = do_system("../../lanforge/lanforge-scripts/openwrt_ctl.py --scheme serial --tty $serial --action sysupgrade --value \"lanforge\@$ap_gw:tip-$jfile\"");
 
       # System should be rebooted at this point.
@@ -197,15 +238,22 @@ for ($i = 0; $i<@lines; $i++) {
       $ap_out = do_system("../../lanforge/lanforge-scripts/openwrt_ctl.py --scheme serial --tty $serial --action reboot");
       print ("Reboot DUT so overlay takes effect:\n$ap_out\n");
 
-      # TODO:  Allow specifying other tests.
+      if ($ttype eq "fast") {
          $ap_out = do_system("cd $tb_dir && DUT_SW_VER=$swver ./run_basic_fast.bash");
-      print("Regression test script output:\n$ap_out\n");
+      }
+      else {
+         $ap_out = do_system("cd $tb_dir && DUT_SW_VER=$swver ./run_basic.bash");
+      }
+      print("Regression $ttype test script output:\n$ap_out\n");
 
       #When complete, upload the results to the requested location.
       if ($ap_out =~ /Results-Dir: (.*)/g) {
          my $rslts_dir = $1;
          print ("Found results at: $rslts_dir\n");
-         do_system("scp -r $rslts_dir $report_to");
+         do_system("mv $rslts_dir /tmp/$report_name");
+         do_system("scp -r /tmp/$report_name $report_to/");
+         do_system("echo $fname > /tmp/NEW_RESULTS-$fname");
+         do_system("scp /tmp/NEW_RESULTS-$fname $report_to/");
       }
 
       exit(0);