wlan-lanforge-scripts/gui/kpi.java

//
// LANforge-GUI Source Code
// Copyright (C) 1999-2020  Candela Technologies Inc
// http://www.candelatech.com
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU Library General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
//
// Contact:  Candela Technologies <support@candelatech.com> if you have any
//           questions.
//

import java.text.*;
import java.util.concurrent.*;
import java.io.*;
import java.net.URL;
import java.util.*;
import java.nio.file.*;

/** Process a set of test results generated by the CI/CD process for a test type in a single testbed.
 * There are currently no automated comparisons done across different testbeds.
 * Generate historical graphs and links to each test run.
 * Each test results directory would be packaged up by the lf_gui_report_summary.pl script,
 * called by the basic_regression.bash automated regression test script.
 * Example:  java kpi --dir /var/www/html/tip/testbeds/ferndale-basic-01/reports/basic
 */
public class kpi {
   String lc_osname;
   String home_dir;

   static final String out_sep = "\t";
   static final String in_sep = "\t";
   
   public static int PRIORITY_IDX = 6;
   public static int TEST_ID_IDX = 7;
   public static int SHORT_DESC_IDX = 8;
   public static int PASS_FAIL_IDX = 9;
   public static int NUMERIC_SCORE_IDX = 10;
   public static int NOTES_IDX = 11;
   public static int UNITS_IDX = 12;
   public static int GRAPH_GROUP_IDX = 13;
   public static int SUBTEST_PASS_IDX = 14;
   public static int SUBTEST_FAIL_IDX = 15;

   public static String TESTBED_TEMPLATE = "testbed_template.html";
   public static String AP_AUTO_BASIC_CX = "ap_auto_basic_cx";

   public static String HTML_COLOR_PASS = "#bee7aa";
   public static String HTML_COLOR_FAIL = "#f9c5c0";
   public static String HTML_COLOR_WARNING = "#f8f6ad";

   public static String PF = "PASS-FAIL";

   public kpi() {
      priv_init();
   }

   static String yellowTd(String txt) {
      return "<td><span style='background:" + HTML_COLOR_WARNING + ";color:black;'>" + txt + "</span></td>";
   }

   static String redTd(String txt) {
      return "<td><span style='background:" + HTML_COLOR_FAIL + ";color:black;'>" + txt + "</span></td>";
   }

   static String greenTd(String txt) {
      return "<td><span style='background:" + HTML_COLOR_PASS + ";color:black;'>" + txt + "</span></td>";
   }

   public boolean is_mac() {
      return lc_osname.startsWith("mac os x");
   }
   public boolean is_win() {
      return lc_osname.startsWith("windows");
   }
   public boolean is_linux() {
      return lc_osname.startsWith("linux");
   }

   private void priv_init() {
      lc_osname = System.getProperty("os.name").toLowerCase();
      home_dir = System.getProperty("user.home");
   }

   public static void main(String[] args) {
      kpi k = new kpi();
      try {
         k.work(args);
      }
      catch (Exception ee) {
         ee.printStackTrace();
      }
   }

   public static String toStringNum(double i, int precision) {
      NumberFormat nf = NumberFormat.getInstance();
      if (0.0 == i) {
         // Don't pad precision on '0', just adds needless clutter.
         return nf.format(i);
      }
      else {
         nf = (NumberFormat)(nf.clone());
         nf.setMaximumFractionDigits(precision);
         nf.setMinimumFractionDigits(precision);
         return nf.format(i);
      }
   }

   public void work(String[] args) {
      String dir = null;
      String results_url = "";
      String caseid = "";
      String slack_fname = "";
      String testbed_name = "LANforge CICD";

      for (int i = 0; i<args.length; i++) {
         if (args[i].equals("--dir")) {
            dir = args[i+1];
         }
         else if (args[i].equals("--caseid")) {
            caseid = args[i+1];
         }
         else if (args[i].equals("--results_url")) {
            results_url = args[i+1];
         }
         else if (args[i].equals("--slack_fname")) {
            slack_fname = args[i+1];
         }
         else if (args[i].equals("--testbed_name")) {
            testbed_name = args[i+1];
         }
         else if (args[i].equals("--help") || args[i].equals("-h") || args[i].equals("-?")) {
            System.out.println("Usage: $0 --dir /path/to/test-collection\n" +
                               "--caseid [case-id]\n" +
                               "--results_url [url]\n" +
                               "--slack_fname [slack url webhook file name]\n" +
                               "--testbed_name [My Testbed]");
            System.exit(0);
         }
      }

      Hashtable<String, String> test_names = new Hashtable();
      Vector<String> test_namesv = new Vector();
      Vector<Run> runs = new Vector();

      test_names.put(PF, PF);
      test_namesv.add(PF);

      try {
         DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(dir));
         for (Path file: stream) {
            File f = file.toFile(); // this is the test run dir
            // Inside of it is individual tests.
            if (!f.isDirectory()) {
               continue;
            }
            //System.out.println("Checking sub-directory/file (run): " + f.getAbsolutePath());
            DirectoryStream<Path> stream2 = Files.newDirectoryStream(file);
            Run run = null;

            for (Path file2: stream2) {
               File f2 = file2.toFile(); // this is the test case dir in the test run
               //System.out.println("Checking test-case directory/file: " + f2.getAbsolutePath());
               // Directory full of test results?
               if (f2.isDirectory()) {
                  DirectoryStream<Path> stream3 = Files.newDirectoryStream(file2);
                  Test test = new Test(f2.getName());
                  if (run == null) {
                     run = new Run(f.getName());
                     runs.add(run);
                  }
                  System.out.println("Run: " + run + " adding test: " + test);
                  run.addTest(test);

                  // TODO:
                  // Add log processing.  In particular, look in console logs for things like:
                  // "ath10k_pci 0000:01:00.0: wmi command 36967 timeout, restarting hardware"
                  // "WARNING: CPU: 0 PID: 8907 at backports-4.

                  for (Path file3: stream3) {
                     File f3 = file3.toFile();
                     String fname = f3.getName();
                     if (fname.equals("kpi.csv")) {
                        try {
                           BufferedReader br = new BufferedReader(new FileReader(f3));
                           if (test_names.get(f2.getName()) == null) {
                              test_names.put(f2.getName(), f2.getName());
                              test_namesv.add(f2.getName());
                           }
                           String line;
                           while ((line = br.readLine()) != null) {
                              test.addLine(line);
                           }
                        }
                        catch (FileNotFoundException enf) {
                           // ignore
                        }
                        catch (Exception e) {
                           e.printStackTrace();
                        }
                     }// if kpi.csv
                     else if (fname.startsWith("kpi-") && fname.endsWith(".png")) {
                        test.addKpiImage(fname);
                     }
                     else if (fname.equals("logs")) {
                        File logs_csv = new File(f3.getAbsolutePath() + File.separator + "logs.csv");
                        if (logs_csv.exists()) {
                           try {
                              BufferedReader br = new BufferedReader(new FileReader(logs_csv));
                              String line;
                              while ((line = br.readLine()) != null) {
                                 test.addLogCsv(line);
                              }
                           }
                           catch (Exception e) {
                              e.printStackTrace();
                           }
                        }
                     }
                  }// for all files in the test dir  
               }
            }
         }
      } catch (IOException | DirectoryIteratorException x) {
         // IOException can never be thrown by the iteration.
         // In this snippet, it can only be thrown by newDirectoryStream.
         System.err.println(x);
      }

      // Sort runs so that earliest is first.
      class SortbyDate implements Comparator<Run> { 
         // Used for sorting in ascending order of 
         // roll number 
         public int compare(Run a, Run b) { 
            long c = a.getDateMs() - b.getDateMs();
            if (c < 0)
               return -1;
            if (c > 0)
               return 1;
            return 0;
         }
      }

      runs.sort(new SortbyDate());

      // Create meta-test results for each run.
      for (int i = 0; i<runs.size(); i++) {
         Run run = runs.elementAt(i);
         Test t = new Test(PF);
         run.addTest(t);
         // Build fake kpi title and add that,
         // then add fake kpi results line
         // Example lines from a kpi.csv
         //Date	test-rig	dut-hw-version	dut-sw-version	dut-model-num	dut-serial-num	test-priority	test-id	short-description	pass/fail	numeric-score	test details	Units	Graph-Group	Subtest-Pass	Subtest-Fail
         //1590190791848	Ferndale-01-Basic	Linksys-EA8300	d88ea40	Linksys-EA8300		90	AP Auto	Basic Client Connectivity Dual-Band MIN		8.0	Minimum CX time (ms)	Time (ms)	CX-Time	0	0

         t.addLine("Date	test-rig	dut-hw-version	dut-sw-version	dut-model-num	dut-serial-num	test-priority	test-id	short-description	pass/fail	numeric-score	test details	Units	Graph-Group	Subtest-Pass	Subtest-Fail");
         t.addLine(run.getDateMs() + kpi.in_sep + run.getTestRig() + kpi.in_sep + run.getDutHwVer() + kpi.in_sep + run.getDutSwVer()
                   + kpi.in_sep + run.getDutModelNum() + kpi.in_sep + run.getDutSerNum() + kpi.in_sep + "100" + kpi.in_sep + PF
                   + kpi.in_sep + "Total Pass" + kpi.in_sep + kpi.in_sep + run.getPass() + kpi.in_sep + "Total passing tests and sub-tests for this run."
                   + kpi.in_sep + "Number" + kpi.in_sep + PF + kpi.in_sep + "0" + kpi.in_sep + "0");
         t.addLine(run.getDateMs() + kpi.in_sep + run.getTestRig() + kpi.in_sep + run.getDutHwVer() + kpi.in_sep + run.getDutSwVer()
                   + kpi.in_sep + run.getDutModelNum() + kpi.in_sep + run.getDutSerNum() + kpi.in_sep + "100" + kpi.in_sep + PF
                   + kpi.in_sep + "Total Failures" + kpi.in_sep + kpi.in_sep + run.getFail() + kpi.in_sep + "Total failing tests and sub-tests for this run."
                   + kpi.in_sep + "Number" + kpi.in_sep + PF + kpi.in_sep + "0" + kpi.in_sep + "0");
      }

      // Link to latest test run that has the test id
      Hashtable<String, Run> test_id_links = new Hashtable();

      // We have read everything into memory.
      // For each test, generate data over time.
      Hashtable<String, History> hist_data = new Hashtable();
      Vector<History> hist_datav = new Vector();
      for (String tname: test_namesv) {
         System.out.println("Checking testname: " + tname + " runs.size: " + runs.size());
         // For each test, find all runs that have this test and consolidate data
         for (int i = 0; i<runs.size(); i++) {
            Run run = runs.elementAt(i);
            Test t = run.findTest(tname);
            System.out.println("Run [" + i + "]  Test: " + t);
            if (t != null) {
               test_id_links.put(tname, run);
               try {
                  History hist = hist_data.get(tname);
                  if (hist == null) {
                     hist = new History(tname);
                     hist_data.put(tname, hist);
                     hist_datav.add(hist);
                     System.out.println("Adding history datav, tname: " + tname);
                  }
                  for (int z = 0; z<t.data.size(); z++) {
                     Row r = t.data.elementAt(z);
                     HistRow csv = hist.findRow(r.getShortDescKey());
                     if (csv == null) {
                        csv = new HistRow(r);
                        hist.addRow(csv);
                        System.out.println("Adding history row: " + r.getShortDescKey());
                     }
                     String score =  t.data.elementAt(z).getScore();
                     csv.csv.append(i + kpi.out_sep + score + System.lineSeparator());
                     csv.scores.add(Double.valueOf(score));
                  }
               }
               catch (Exception eee) {
                  eee.printStackTrace();
               }
            }
         }
      }

      StringBuffer scores = new StringBuffer();
      StringBuffer plots = new StringBuffer();
      StringBuffer groups = new StringBuffer();
      StringBuffer runs_rows = new StringBuffer();

      // For all per-test history, print out csv files
      for (History hist : hist_datav) {
         String hk = hist.getName();
         Hashtable<String, Vector> groupsh = new Hashtable();
         Vector<String> groupsn = new Vector();
         System.out.println("Checking history-data: " + hk);
         for (HistRow csv : hist.csv) {
            String ck = csv.getName();
            String title = csv.getTitle();
            String units = csv.getUnits();
            String g = csv.getGraphGroup();

            System.out.println("csv name: " + ck);

            if (!(g.equals("NA") || units.equals(""))) {
               Vector<HistRow> gv = groupsh.get(g);
               if (gv == null) {
                  gv = new Vector();
                  groupsh.put(g, gv);
                  groupsn.add(g);
               }
               gv.add(csv);
            }

            if (units.equals("NA") || units.equals("")) {
               units = "Data";
            }
            try {
               String cf = dir + File.separator + hk + "::" + ck + ".csv";
               System.out.println("Creating gnuplot csv: " + cf);
               FileWriter f = new FileWriter(cf);
               f.write(csv.csv.toString());
               f.close();
               csv.setFname(cf);

               ShellExec exec = new ShellExec(true, true);
               int rv = exec.execute("gnuplot", null, true,
                                     "-e", "set ylabel \"" + units + "\"",
                                     "-e", "filename='" + cf + "'",
                                     "-e", "set title '" + title + "'",
                                     "default.plot");
               if (rv != 0) {
                  System.out.println("gnuplot for filename: " + cf + " rv: " + rv);
                  System.out.println(exec.getOutput());
                  System.out.println(exec.getError());
               }

               File png = new File("plot.png");
               if (!png.exists()) {
                  System.out.println("ERROR:  File: " + png + " does not exist.");
               }
               String npng = hk + "::" + ck + ".png";
               String nplot_name = dir + File.separator + npng;
               System.out.println("Rename plot: " + png + " to: " + nplot_name);
               File nf = new File(nplot_name);
               Path nfp = Files.move(png.toPath(), nf.toPath(), StandardCopyOption.REPLACE_EXISTING);//boolean renamed = png.renameTo(nf);
               if (!nf.exists()) {
                  System.out.println("ERROR:  New file: " + nf + " does not exist.");
               }

               exec = new ShellExec(true, true);
               rv = exec.execute("gnuplot", null, true, "-e", "filename='" + cf + "'",
                                     "mini.plot");
               if (rv != 0) {
                  System.out.println("mini gnuplot for filename: " + cf + " rv: " + rv);
                  System.out.println(exec.getOutput());
                  System.out.println(exec.getError());
               }

               png = new File("plot.png");
               String npngt = hk + "::" + ck + "-thumb.png";
               Files.move(png.toPath(), new File(dir + File.separator + npngt).toPath(), StandardCopyOption.REPLACE_EXISTING);
               
               String hk_str = hk;
               Run last = test_id_links.get(hk);
               if (!hk.equals(PF)) {
                  if (last != null) {
                     hk_str = "<a href=\"" + last.getName() + "/" + hk + "/index.html\">" + hk + "</a>";
                  }
               }

               StringBuffer change = new StringBuffer();
               Double cur = csv.scores.elementAt(csv.scores.size() - 1);
               change.append("Last Value: " + toStringNum(cur, 2));
               if (csv.scores.size() > 1) {
                  Double prev = csv.scores.elementAt(csv.scores.size() - 2);
                  change.append("<br>Delta for last run: " + toStringNum((cur - prev), 2));
                  if (cur != 0) {
                     change.append("<br>Percentage change: " + toStringNum(100.0 * ((cur - prev) / cur), 2) + "%");
                  }
               }

               String row_str = ("<tr><td>" + hk_str + "</td><td>" + title + "</td><td><a href=\"./" + npng + "\"><img src=\"./" + npngt + "\"></a></td><td>"
                                 + change + "</td></tr>\n");
               if (csv.getPriority() >= 100) {
                  scores.append(row_str);
               }
               else {
                  plots.append(row_str);
               }
            }
            catch (Exception ee) {
               ee.printStackTrace();
            }
         }

         // Graph groups
         for (String g : groupsn) {
            Vector<HistRow> datasets = groupsh.get(g);
            if (datasets.size() < 2) {
               continue; // group of 1 doesn't count
            }

            HistRow csv0 = datasets.elementAt(0);
            String title = g;
            String units = csv0.getUnits();

            // Don't group scores
            if (units.equalsIgnoreCase("Score")) {
               continue;
            }

            if (units.equals("NA") || units.equals("")) {
               units = "Data";
            }

            System.out.println("title: " + title + " units: " + units);
            try {
               StringBuffer plot = new StringBuffer("plot ");
               StringBuffer mplot = new StringBuffer("plot ");
               boolean first = true;
               for (HistRow csv: datasets) {
                  if (!first) {
                     plot.append(", ");
                     mplot.append(", ");
                  }
                  plot.append("\'" + csv.getFname() + "\' using 1:2 with lines title \'" + csv.getName().replace("_", " ") + "\'");
                  mplot.append("\'" + csv.getFname() + "\' using 1:2 with lines notitle");
                  first = false;
               }

               BufferedReader br = new BufferedReader(new FileReader("default_group.plot"));
               FileWriter f = new FileWriter("tmp.plot");
               String line;
               while ((line = br.readLine()) != null) {
                  f.write(line);
                  f.write(System.lineSeparator());
               }
               f.write(plot.toString());
               f.write(System.lineSeparator());
               f.close();
               br.close();

               //System.out.println("group plot: " + plot);
               ShellExec exec = new ShellExec(true, true);
               int rv = exec.execute("gnuplot", null, true,
                                     "-e", "set ylabel '" + units + "'",
                                     "-e", "set title '" + title + "'",
                                     "tmp.plot");
               if (rv != 0) {
                  System.out.println("gnuplot for group: " + title + " rv: " + rv);
                  System.out.println(exec.getOutput());
                  System.out.println(exec.getError());
               }

               File png = new File("plot.png");
               String npng = hk + "::" + g + ".png";
               Files.move(png.toPath(), new File(dir + File.separator + npng).toPath(), StandardCopyOption.REPLACE_EXISTING);

               br = new BufferedReader(new FileReader("mini_group.plot"));
               f = new FileWriter("tmp.plot");
               while ((line = br.readLine()) != null) {
                  f.write(line);
                  f.write(System.lineSeparator());
               }
               f.write(mplot.toString());
               f.write(System.lineSeparator());
               f.close();
               br.close();

               exec = new ShellExec(true, true);
               rv = exec.execute("gnuplot", null, true,
                                 "tmp.plot");
               if (rv != 0) {
                  System.out.println("mini gnuplot for group: " + title + " rv: " + rv);
                  System.out.println(exec.getOutput());
                  System.out.println(exec.getError());
               }

               png = new File("plot.png");
               String npngt = hk + "::" + g + "-thumb.png";
               Files.move(png.toPath(), new File(dir + File.separator + npngt).toPath(), StandardCopyOption.REPLACE_EXISTING);

               String hk_str = hk;
               Run last = test_id_links.get(hk);
               if (!hk.equals(PF)) {
                  if (last != null) {
                     hk_str = "<a href=\"" + last.getName() + "/" + hk + "/index.html\">" + hk + "</a>";
                  }
               }

               groups.append("<tr><td>" + hk_str + "</td><td>" + title + "</td><td><a href=\"./" + npng + "\"><img src=\"./" + npngt + "\"></a></td></tr>\n");

            }
            catch (Exception ee) {
               ee.printStackTrace();
            }
         }
         
      }

      String test_bed = "Test Bed";
      String last_run = "";
      String last_run_kpi_images = "";
      StringBuffer pngs = new StringBuffer();

      boolean cp = true;
      for (int i = 0; i<runs.size(); i++) {
         Run run = runs.elementAt(i);
         test_bed = run.getTestRig();
         StringBuffer logs_str = new StringBuffer();
         int logv = run.getLogBugs();
         if (logv > 0) {
            logs_str.append(redTd(logv + ""));
         }
         else {
            logs_str.append(greenTd(logv + ""));
         }

         logv = run.getLogWarnings();
         if (logv > 0) {
            logs_str.append(redTd(logv + ""));
         }
         else {
            logs_str.append(greenTd(logv + ""));
         }

         logv = run.getLogCrashes();
         if (logv > 0) {
            logs_str.append(redTd(logv + ""));
         }
         else {
            logs_str.append(greenTd(logv + ""));
         }

         logv = run.getLogRestarting();
         if (logv > 0) {
            logs_str.append(redTd(logv + ""));
         }
         else {
            logs_str.append(greenTd(logv + ""));
         }

         String row_text = ("<tr><td>" + i + "</td><td><a href=\"" + run.getName() + "/index.html\">" + run.getName() + "</a></td><td>" + run.getDate()
                            + "</td><td>" + run.getDutHwVer() + "</td><td>" + run.getDutSwVer()
                            + "</td><td>" + run.getDutModelNum() + "</td>" + greenTd(run.getPass() + "") + redTd(run.getFail() + "") + logs_str + "</tr>\n");
            
         if (i == (runs.size() - 1)) {
            // Last run
            int png_row_count = 0;
            boolean needs_tr = true;
            last_run = row_text;
            for (Test t : run.testsv) {
               for (String png : t.kpi_images) {
                  if (png.indexOf("-print") >= 0) {
                     continue; // skip the print variants of the image.
                  }
                  if (needs_tr) {
                     pngs.append("<tr>");
                     needs_tr = false;
                  }
                  String img_title = "Test: " + t.getName();
                  String fname = run.getName() + "/" + t.getName() + "/" + png;
                  pngs.append("<td><a href=\"" + fname + "\"><img src=\"" + fname + "\" style='width:400px;max-width:400px' title=\""
                              + img_title + "\"></a></td>\n");
                  png_row_count++;
                  if (png_row_count == 2) {
                     png_row_count = 0;
                     pngs.append("</tr>\n");
                     needs_tr = true;
                  }
               }
            }

            if ((!needs_tr) && pngs.length() > 0) {
               pngs.append("</tr>\n");
            }

            String testrails_msg = ("MSG Run: " + run.getName() + " Date: " + run.getDate() + " DUT-HW: " + run.getDutHwVer() + "DUT-SW: " + run.getDutSwVer()
                                    + " Passed: " + run.getPass() + " Fail: " + run.getFail()
                                    + " Log-Bugs: " + run.getLogBugs() + " Log-Warnings: " + run.getLogWarnings()
                                    + " Log-Crashes: " + run.getLogCrashes() + " Log-Restarting: " + run.getLogRestarting());

            if (!caseid.equals("")) {
               try {
                  // Create file for propagating results into external tool
                  String ofile = dir + File.separator + "testrails.txt";
                  BufferedWriter bw = new BufferedWriter(new FileWriter(ofile));
             
                  bw.write("CASE_ID " + caseid);
                  bw.write(System.lineSeparator());
                  bw.write("RUN_ID " + i);
                  bw.write(System.lineSeparator());
                  bw.write(testrails_msg);
                  bw.write(System.lineSeparator());
                  bw.write("MSG URL: " + results_url + "/" + run.getName());
                  bw.write(System.lineSeparator());

                  bw.close();

                  System.out.println("See testrails results: " + ofile);
               }
               catch (Exception eee) {
                  eee.printStackTrace();
               }
            }

            if (!slack_fname.equals("")) {
               String slack_content_type = "Content-type: application/json";
               String slack_msg = "{\"text\":\"<" + results_url + "|" + testbed_name + ">"
                  + " Results: <" + results_url + "/" + run.getName() + "|" + run.getName() + ">"
                  + " Errors: " + (run.getLogCrashes() + run.getLogRestarting() + run.getLogBugs()) + " Warnings: " + run.getLogWarnings()
                  + "\"}";
               try {
                  BufferedReader br = new BufferedReader(new FileReader(slack_fname));
                  String slack = br.readLine();
                  String cmd = "curl -X POST -H '" + slack_content_type + "' --data '" + slack_msg + "' " + slack;
                  System.out.println(cmd);

                  ShellExec exec = new ShellExec(true, true);
                  int rv = exec.execute("curl", null, true,
                                        "-X", "POST",
                                        "-H", slack_content_type,
                                        "--data", slack_msg, slack);
                  if (rv != 0) {
                     System.out.println("curl slack post of msg: " + slack_msg + " to: " + slack + " failed.\n");
                     System.out.println(exec.getOutput());
                     System.out.println(exec.getError());
                  }
               }
               catch (Exception eeee) {
                  System.out.println("slack_msg: " + slack_msg + " slack_fname: " + slack_fname);
                  eeee.printStackTrace();
               }
            }
         }

         runs_rows.append(row_text);

         if (cp) {
            try {
               String fname;
               copy("CandelaLogo2-90dpi-200x90-trans.png", dir + File.separator + run.getName(), dir);
               copy("candela_swirl_small-72h.png", dir + File.separator + run.getName(), dir);
               copy("canvil.ico", dir + File.separator + run.getName(), dir);
               copy("custom.css", dir + File.separator + run.getName(), dir);
               copy("report.css", dir + File.separator + run.getName(), dir);

               cp = false;
            }
            catch (Exception ee) {
               ee.printStackTrace();
            }
         }
      }

      try {
         // Read in the testbed_template.html and update it with our info
         BufferedReader br = new BufferedReader(new FileReader(new File(kpi.TESTBED_TEMPLATE)));
         String ofile = dir + File.separator + "index.html";
         BufferedWriter bw = new BufferedWriter(new FileWriter(ofile));
         String line;
         while ((line = br.readLine()) != null) {
            line = line.replace("___TITLE___", test_bed + " Report History");
            line = line.replace("___SCORE_RUNS___", scores.toString());
            line = line.replace("___GROUP_GRAPHS___", groups.toString());
            line = line.replace("___DATA_GRAPHS___", plots.toString());
            line = line.replace("___TEST_RUNS___", runs_rows.toString());
            line = line.replace("___LATEST_RUN___", last_run);
            line = line.replace("___LATEST_RUN_PNGS___", pngs.toString());
            bw.write(line);
            bw.write(System.lineSeparator());
         }
         
         br.close();
         bw.close();

         System.out.println("See " + ofile);
      }
      catch (Exception eee) {
         eee.printStackTrace();
      }
   } // ~work()

   public void copy(String fname, String from, String to) throws Exception {
      Path fromp = Paths.get(from + File.separator + fname);
      Path top = Paths.get(to + File.separator + fname);
      Files.copy(fromp, top, StandardCopyOption.REPLACE_EXISTING);
   }
}


/** This holds a datapoint for a particular test result for each of the Runs.
 * This is used to generate historical graphs and comparisons.
 * If a test generates two KPI results, there will be two HistRows for that
 * test case.
 */
class HistRow {
   String fname;
   String name;
   String title = "";
   String units = "";
   String graph_group = "";
   int prio = 0;
   StringBuffer csv = new StringBuffer();
   Vector<Double> scores = new Vector();

   public HistRow(Row r) {
      name = r.getShortDescKey();
      title = r.getShortDesc();
      units = r.getUnits();
      graph_group = r.getGraphGroup();
      prio = r.getPriority();
   }

   int getPriority() {
      return prio;
   }

   String getFname() {
      return fname;
   }

   void setFname(String s) {
      fname = s;
   }

   String getName() {
      return name;
   }

   String getTitle() {
      return title;
   }

   String getUnits() {
      return units;
   }

   String getGraphGroup() {
      return graph_group;
   }
}

/** This holds all HistRow objects for each test name for each run.
 * It is used for generating historical graphs and comparisons.
 */
class History {
   String name;
   Vector<HistRow> csv = new Vector();
   Hashtable<String, HistRow> csvh = new Hashtable(); // lookup by name

   public History(String n) {
      name = n;
   }

   public String getName() {
      return name;
   }

   HistRow findRow(String n) {
      //System.out.println("findCsv, n: " + n);
      return csvh.get(n);
   }

   void addRow(HistRow r) {
      csv.add(r);
      csvh.put(r.getName(), r);
   }
}

/** A Row represents a single KPI csv data point.  The csv is split into tokens
 * for easier manipulation by other code.
 */
class Row {
   Vector<String> rdata = new Vector();
   String short_desc_key = null;

   String getNotes() {
      return rdata.elementAt(kpi.NOTES_IDX);
   }

   String getSubtestPassed() {
      if (rdata.size() > kpi.SUBTEST_PASS_IDX)
         return rdata.elementAt(kpi.SUBTEST_PASS_IDX);
      return "";
   }

   String getSubtestFailed() {
      if (rdata.size() > kpi.SUBTEST_FAIL_IDX)
         return rdata.elementAt(kpi.SUBTEST_FAIL_IDX);
      return "";
   }

   String getUnits() {
      try {
         return rdata.elementAt(kpi.UNITS_IDX);
      }
      catch (Exception e) {
         return "";
      }
   }

   String getGraphGroup() {
      try {
         return rdata.elementAt(kpi.GRAPH_GROUP_IDX);
      }
      catch (Exception e) {
         return "";
      }
   }

   int getPriority() {
      try {
         return Long.valueOf(rdata.elementAt(kpi.PRIORITY_IDX)).intValue();
      }
      catch (Exception e) {
         return 0;
      }
   }

   String getPassFail() {
      return rdata.elementAt(kpi.PASS_FAIL_IDX);
   }

   String getScore() {
      return rdata.elementAt(kpi.NUMERIC_SCORE_IDX);
   }

   String getShortDesc() {
      return rdata.elementAt(kpi.SHORT_DESC_IDX);
   }

   String getTestId() {
      return rdata.elementAt(kpi.TEST_ID_IDX);
   }

   String getShortDescKey() {
      return short_desc_key;
   }

   void setShortDescKey(String s) {
      short_desc_key = s;
   }

   public String toString() {
      StringBuffer sb = new StringBuffer();
      sb.append("Row " + getShortDescKey() + "  ");
      for (int i = 0; i<rdata.size(); i++) {
         sb.append("[" + i + "] == " + rdata.elementAt(i) + "  ");
      }
      return sb.toString();
   }
}

/** A test contains information on one executation of one test.  For instance,
 * a wifi-capacity-test would be a single test.  It may create a KPI.csv file with
 * multiple rows.  It may also have logs for this test run, and KPI graph images.
 * A Run consists of multiple tests.
 */
class Test {
   String name;
   Vector<String> titles = null;
   Vector<Row> data = new Vector();
   Hashtable<String, String> descs = new Hashtable();
   Vector<String> kpi_images = new Vector();
   Vector<String> log_csv = new Vector();
   int pass = 0;
   int fail = 0;

   int log_bugs = 0;
   int log_warnings = 0;
   int log_crashes = 0;
   int log_restarting = 0;

   long date_ms = 0;
   public String date = "NA";
   public String test_rig = "NA";
   public String dut_hw_version = "NA";
   public String dut_sw_version = "NA";
   public String dut_model_num = "NA";
   public String dut_serial_num = "NA";

   public Test(String n) {
      name = n;
   }

   public int getLogBugs() {
      return log_bugs;
   }

   public int getLogWarnings() {
      return log_warnings;
   }

   public int getLogCrashes() {
      return log_crashes;
   }

   public int getLogRestarting() {
      return log_restarting;
   }

   public String toString() {
      return "Name: " + name;
   }

   public int getPass() {
      return pass;
   }

   public int getFail() {
      return fail;
   }

   void addKpiImage(String s) {
      kpi_images.add(s);
   }

   long getDateMs() {
      return date_ms;
   }

   String getTestRig() {
      return test_rig;
   }

   String getDutHwVer() {
      return dut_hw_version;
   }

   String getDutSwVer() {
      return dut_sw_version;
   }

   String getDutSerialNum() {
      return dut_serial_num;
   }

   String getDutModelNum() {
      //System.out.println("Test: " + getName() + " model-num: " + dut_model_num);
      return dut_model_num;
   }

   String getDate() {
      return date;
   }

   String getName() {
      return name;
   }

   void addLogCsv(String l) {
      log_csv.add(l);
      try {
         StringTokenizer st = new StringTokenizer(l, "\t");
         String tok = st.nextToken();
         if (tok.equals("FILE")) {
            // title, ignore rest of this title
            return;
         }
         log_bugs += Long.valueOf(st.nextToken());
         log_warnings += Long.valueOf(st.nextToken());
         log_crashes += Long.valueOf(st.nextToken());
         log_restarting += Long.valueOf(st.nextToken());
      }
      catch (Exception e) {
         e.printStackTrace();
      }
   }

   void addLine(String l) {
      if (titles == null) {
         titles = new Vector();
         StringTokenizer st = new StringTokenizer(l, kpi.in_sep, true);
         boolean last_was_sep = false;
         while (st.hasMoreTokens()) {
            String tok = st.nextToken();
            if (tok.equals(kpi.in_sep)) {
               if (last_was_sep) {
                  titles.add(new String());
               }
               last_was_sep = true;
            }
            else {
               titles.add(tok);
               last_was_sep = false;
            }
         }
      }
      else {
         Row row = new Row();
         data.add(row);
         StringTokenizer st = new StringTokenizer(l, kpi.in_sep, true);
         int idx = 0;
         System.out.println("new line: " + l);
         boolean last_was_sep = false;
         while (st.hasMoreTokens()) {
            String rtok = st.nextToken();
            if (rtok.equals(kpi.in_sep)) {
               if (last_was_sep) {
                  row.rdata.add(new String());
                  idx++;
               }
               last_was_sep = true;
            }
            else {
               row.rdata.add(rtok);
               idx++;
               last_was_sep = false;
            }

            if ((data.size() >= 1) && (!last_was_sep) && dut_sw_version.equals("NA")) { // first row is being added
               if (titles.elementAt(idx - 1).equalsIgnoreCase("test-rig")) {
                  test_rig = rtok;
               }
               else if (titles.elementAt(idx - 1).equalsIgnoreCase("dut-hw-version")) {
                  dut_hw_version = rtok;
               }
               else if (titles.elementAt(idx - 1).equalsIgnoreCase("dut-sw-version")) {
                  dut_sw_version = rtok;
               }
               else if (titles.elementAt(idx - 1).equalsIgnoreCase("dut-model-num")) {
                  dut_model_num = rtok;
               }
               else if (titles.elementAt(idx - 1).equalsIgnoreCase("Date")) {
                  //System.out.println("idx: " + idx + " rtok: " + rtok);
                  date_ms = Long.valueOf(rtok).longValue();
                  date = new Date(date_ms).toString();
               }
            }
            //System.out.println("idx: " + idx);
         }
         //System.out.println("done tok reading loop");

         String pf = row.getPassFail().toLowerCase();
         if (pf.indexOf("pass") >= 0) {
            pass++;
         }
         else if (pf.indexOf("fail") >= 0) {
            fail++;
         }

         String spass = row.getSubtestPassed();
         try {
            if (!spass.equals("")) {
               pass += Long.valueOf(spass).longValue();
            }
         }
         catch (Exception ee) {
            ee.printStackTrace();
         }

         String sfail = row.getSubtestFailed();
         try {
            if (!sfail.equals("")) {
               fail += Long.valueOf(sfail).longValue();
            }
         }
         catch (Exception ee) {
            ee.printStackTrace();
         }

         row.setShortDescKey(row.getShortDesc().replace(" ", "_"));
         //System.out.println("Row: " + row);
         descs.put(row.getShortDesc(), row.getShortDesc());
      }
   }//addline
}//Test


/** A Run is a collection of tests.  This class encompasses the entire regression test
 * for a particular flavor of test (basic, fast, etc).
 * The CI/CD process will create one or more Runs per build artifact per testbed.
 */
class Run {
   String name;
   Hashtable<String, Test> tests = new Hashtable();
   Vector<Test> testsv = new Vector();

   public Run(String n) {
      name = n;
   }

   public String toString() {
      return "Name: " + name;
   }

   int getPass() {
      int pass = 0;
      for (Test t: testsv) {
         pass += t.getPass();
      }
      return pass;
   }

   int getFail() {
      int fail = 0;
      for (Test t: testsv) {
         fail += t.getFail();
      }
      return fail;
   }

   int getLogBugs() {
      int fail = 0;
      for (Test t: testsv) {
         fail += t.getLogBugs();
      }
      return fail;
   }

   int getLogWarnings() {
      int fail = 0;
      for (Test t: testsv) {
         fail += t.getLogWarnings();
      }
      return fail;
   }

   int getLogCrashes() {
      int fail = 0;
      for (Test t: testsv) {
         fail += t.getLogCrashes();
      }
      return fail;
   }

   int getLogRestarting() {
      int fail = 0;
      for (Test t: testsv) {
         fail += t.getLogRestarting();
      }
      return fail;
   }

   Test getFirstTest() {
      return testsv.elementAt(0);
   }

   String getDate() {
      Test t = getFirstTest();
      if (t != null)
         return t.getDate();
      return "";
   }

   long getDateMs() {
      Test t = getFirstTest();
      if (t != null)
         return t.getDateMs();
      return 0;
   }

   String getTestRig() {
      Test t = getFirstTest();
      if (t != null)
         return t.getTestRig();
      return "";
   }

   String getDutHwVer() {
      Test t = getFirstTest();
      if (t != null)
         return t.getDutHwVer();
      return "";
   }

   String getDutSwVer() {
      Test t = getFirstTest();
      if (t != null)
         return t.getDutSwVer();
      return "";
   }

   String getDutSerNum() {
      Test t = getFirstTest();
      if (t != null)
         return t.getDutSerialNum();
      return "";
   }

   String getDutModelNum() {
      Test t = getFirstTest();
      if (t != null)
         return t.getDutModelNum();
      return "";
   }

   String getName() {
      return name;
   }

   void addTest(Test t) {
      tests.put(t.getName(), t);
      testsv.add(t);
   }

   Test findTest(String n) {
      return tests.get(n);
   }
}//Run


// From: https://stackoverflow.com/questions/882772/capturing-stdout-when-calling-runtime-exec

/**
 * Execute external process and optionally read output buffer.
 */
class ShellExec {
   private int exitCode;
   private boolean readOutput, readError;
   private StreamGobbler errorGobbler, outputGobbler;

   public ShellExec() { 
      this(false, false);
   }

   public ShellExec(boolean readOutput, boolean readError) {
      this.readOutput = readOutput;
      this.readError = readError;
   }

   /**
    * Execute a command.
    * @param command   command ("c:/some/folder/script.bat" or "some/folder/script.sh")
    * @param workdir   working directory or NULL to use command folder
    * @param wait  wait for process to end
    * @param args  0..n command line arguments
    * @return  process exit code
    */
   public int execute(String command, String workdir, boolean wait, String...args) throws IOException {
      String[] cmdArr;
      StringBuffer dbg = new StringBuffer();
      dbg.append("Command: " + command);
      if (args != null && args.length > 0) {
         cmdArr = new String[1+args.length];
         cmdArr[0] = command;
         System.arraycopy(args, 0, cmdArr, 1, args.length);
         for (int i = 0; i<args.length; i++) {
            dbg.append(" " + args[i]);
         }
      } else {
         cmdArr = new String[] { command };
      }

      System.out.println("Running cmd: " + dbg.toString());

      ProcessBuilder pb =  new ProcessBuilder(cmdArr);
      File workingDir = (workdir==null ? new File(command).getParentFile() : new File(workdir) );
      pb.directory(workingDir);

      Process process = pb.start();

      // Consume streams, older jvm's had a memory leak if streams were not read,
      // some other jvm+OS combinations may block unless streams are consumed.
      errorGobbler  = new StreamGobbler(process.getErrorStream(), readError);
      outputGobbler = new StreamGobbler(process.getInputStream(), readOutput);
      errorGobbler.start();
      outputGobbler.start();

      exitCode = 0;
      if (wait) {
         try { 
            process.waitFor();
            exitCode = process.exitValue();                 
         } catch (InterruptedException ex) { }
      }
      return exitCode;
   }   

   public int getExitCode() {
      return exitCode;
   }

   public boolean isOutputCompleted() {
      return (outputGobbler != null ? outputGobbler.isCompleted() : false);
   }

   public boolean isErrorCompleted() {
      return (errorGobbler != null ? errorGobbler.isCompleted() : false);
   }

   public String getOutput() {
      return (outputGobbler != null ? outputGobbler.getOutput() : null);        
   }

   public String getError() {
      return (errorGobbler != null ? errorGobbler.getOutput() : null);        
   }

   //********************************************
   //********************************************    

    /**
     * StreamGobbler reads inputstream to "gobble" it.
     * This is used by Executor class when running 
     * a commandline applications. Gobblers must read/purge
     * INSTR and ERRSTR process streams.
     * http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html?page=4
     */
   private class StreamGobbler extends Thread {
      private InputStream is;
      private StringBuilder output;
      private volatile boolean completed; // mark volatile to guarantee a thread safety

      public StreamGobbler(InputStream is, boolean readStream) {
         this.is = is;
         this.output = (readStream ? new StringBuilder(256) : null);
      }

      public void run() {
         completed = false;
         try {
            String NL = System.getProperty("line.separator", "\r\n");

            InputStreamReader isr = new InputStreamReader(is);
            BufferedReader br = new BufferedReader(isr);
            String line;
            while ( (line = br.readLine()) != null) {
               if (output != null)
                  output.append(line + NL); 
            }
         } catch (IOException ex) {
            // ex.printStackTrace();
         }
         completed = true;
      }

      /**
       * Get inputstream buffer or null if stream
       * was not consumed.
       * @return
       */
      public String getOutput() {
         return (output != null ? output.toString() : null);
      }

      /**
       * Is input stream completed.
       * @return
       */
      public boolean isCompleted() {
         return completed;
      }

   }

}