Add failure message when a KPI is not found in a folder for GhostRequest

Automatic script to run influx, grafana, and ghost from wifi capacity and dataplane.

Minor fix in csv_to_influx for correcting missing values

Signed-off-by: Matthew Stidham <stidmatt@gmail.com>

py-dashboard/GhostRequest.py

@@ -291,8 +291,7 @@ class GhostRequest:
                 target_folders.append(folder)
 
         testbeds = list()
-        webpages = self.webpages
-        pdfs = self.pdfs
+        webpagesandpdfs = list()
         high_priority_list = list()
         low_priority_list = list()
         images = list()
@@ -304,6 +303,7 @@ class GhostRequest:
                 target_file = '%s/kpi.csv' % target_folder
                 df = csvreader.read_csv(file=target_file, sep='\t')
                 test_rig = csvreader.get_column(df, 'test-rig')[0]
+                test_id = csvreader.get_column(df, 'test-id')[0]
                 pass_fail = Counter(csvreader.get_column(df, 'pass/fail'))
                 test_pass_fail.append(pass_fail)
                 dut_hw = csvreader.get_column(df, 'dut-hw-version')[0]
@@ -323,11 +323,6 @@ class GhostRequest:
                     text = text + 'Tests passed: 0<br />' \
                                   'Tests failed : 0<br />' \
                                   'Percentage of tests passed: Not Applicable<br />'
-
-            except:
-                print("Failure")
-                target_folders.remove(target_folder)
-                break
                 testbeds.append(test_rig)
                 if testbed is None:
                     testbed = test_rig
@@ -356,15 +351,20 @@ class GhostRequest:
                     pass
                 scp_push.put(target_folder, local_path, recursive=True)
                 files = sftp.listdir(local_path + '/' + target_folder)
+                pdfs = list()
+                webpages = list()
                 for file in files:
                     if 'pdf' in file:
                         url = 'http://%s/%s/%s/%s/%s/%s' % (
                             ghost_host, customer.strip('/'), testbed, test_run, target_folder, file)
-                    pdfs.append('PDF of results: <a href="%s">%s</a>' % (url, file))
+                        pdfs.append('<a href="%s">PDF</a>' % url)
                 if 'index.html' in files:
                     url = 'http://%s/%s/%s/%s/%s/%s' % (
                         ghost_host, customer.strip('/'), testbed, test_run, target_folder, 'index.html')
-                webpages.append('Results webpage: <a href="%s">%s</a><br />' % (url, target_folder))
+                    webpages.append('<a href="%s">HTML</a>' % url)
+                webpagesandpdfsappend = dict()
+                webpagesandpdfsappend[test_id] = pdfs + webpages
+                webpagesandpdfs.append(webpagesandpdfsappend)
                 scp_push.close()
                 self.upload_images(target_folder)
                 for image in self.images:
@@ -377,6 +377,11 @@ class GhostRequest:
                                                      'test-priority'])
 
                 results[0] = ['Short Description', 'Score', 'Test Details', 'Pass or Fail', 'test-priority']
+                for row in results:
+                    try:
+                        row[1] = round(float(row[1]), 2)
+                    except:
+                        pass
 
                 low_priority = csvreader.filter_df(results, 'test-priority', 'less than', 94)
                 high_priority = csvreader.filter_df(results, 'test-priority', 'greater than or equal to', 95)
@@ -384,6 +389,13 @@ class GhostRequest:
 
                 low_priority_list.append(low_priority)
 
+            except:
+                print("Failure")
+                target_folders.remove(target_folder)
+                failuredict = dict()
+                failuredict[target_folder] = ['Failure']
+                webpagesandpdfs.append(failuredict)
+
 
         test_pass_fail_results = sum((Counter(test) for test in test_pass_fail), Counter())
 
@@ -473,10 +485,14 @@ class GhostRequest:
         dut_table = dut_table + '</tbody></table>'
         text = text + dut_table
 
-        for article in zip(pdfs, webpages):
-            if self.debug:
-                print(article)
-            text = text + article[0] + ' | ' + article[1]
+        for dictionary in webpagesandpdfs:
+            text += list(dictionary.keys())[0] + ' report: '
+            for value in dictionary.values():
+                for webpage in value:
+                    text += webpage
+                    if value.index(webpage) + 1 != len(value):
+                        text += ' | '
+            text += '<br />'
 
         for image in images:
             text = text + image

py-scripts/csv_to_influx.py

@@ -57,8 +57,6 @@ class CSVtoInflux():
     # Submit data to the influx db if configured to do so.
     def post_to_influx(self):
         df = self.read_csv(self.target_csv)
-        for row in df:
-            row =[sub.replace('NaN','0') for sub in row]
         length = list(range(0, len(df[0])))
         columns = dict(zip(df[0], length))
         print('columns: %s' % columns)
@@ -68,6 +66,7 @@ class CSVtoInflux():
                          'dut-hw-version', 'dut-sw-version', 'dut-serial-num', 'test-rig', 'Units']
         csv_vs_influx = dict(zip(csv_variables, influx_variables))
         for row in df[1:]:
+            row = [sub.replace('NaN', '0') for sub in row]
             tags = dict()
             print("row: %s" % row)
             short_description = row[columns['short-description']]

py-scripts/influxgrafanaghost.sh

@@ -0,0 +1,46 @@
+#! /bin/bash
+
+MGR=192.168.100.213
+LFUSER=lanforge
+LOCALDIR=/home/matthew/Documents/lanforge-scripts/py-scripts/lftest
+TESTRIG="Matthew-ct523c"
+
+GHOSTTOKEN=60df4b0175953f400cd30650:d50e1fabf9a9b5d3d30fe97bc3bf04971d05496a89e92a169a0d72357c81f742
+
+INFLUXTOKEN=31N9QDhjJHBu4eMUlMBwbK3sOjXLRAhZuCzZGeO8WVCj-xvR8gZWWvRHOcuw-5RHeB7xBFnLs7ZV023k4koR1A==
+INFLUXHOST=c7-grafana.candelatech.com
+
+GRAFANATOKEN=eyJrIjoiS1NGRU8xcTVBQW9lUmlTM2dNRFpqNjFqV05MZkM0dzciLCJuIjoibWF0dGhldyIsImlkIjoxfQ==
+
+rm -r ${LOCALDIR}
+
+mkdir ${LOCALDIR}
+
+./scenario.py --mgr ${MGR} --load BLANK
+
+sleep 10s
+
+./create_l3.py --mgr ${MGR} --num_stations 4 --ssid stidmatt --password stidmatt --security wpa2 --radio wiphy0
+
+./lf_dataplane_test.py --mgr ${MGR} --lf_user ${LFUSER} --lf_password lanforge --instance_name wct_instance \
+--config_name 64_stations --upstream 1.1.eth1 --influx_host c7-grafana.candelatech.com --influx_org Candela \
+--influx_token ${INFLUXTOKEN} --influx_bucket stidmatt --test_rig {TESTRIG} --influx_tag testbed ${TESTRIG} \
+--station 1.1.sta0000 --set DUT_NAME linksys-8450 --local_lf_report_dir ${LOCALDIR} \
+--pull_report \
+--download_speed 85% --upload_speed 0 \
+--raw_line 'cust_pkt_sz: 88 1200' \
+--raw_line 'directions: DUT Transmit;DUT Receive' \
+--raw_line 'traffic_types: UDP' \
+--test_rig Testbed-01 --pull_report
+#--raw_line 'pkts: Custom;60;142;256;512;1024;MTU'
+
+./lf_wifi_capacity_test.py --mgr ${MGR} --lf_user ${LFUSER} --lf_password lanforge --instance_name linksys-8450 \
+--config_name wifi_config --upstream 1.1.eth1 --radio wiphy0 --ssid lanforge --paswd lanforge --security wpa2 \
+--influx_host ${INFLUXHOST} --influx_org Candela --influx_bucket stidmatt --test_rig ${TESTRIG} \
+--influx_token ${INFLUXTOKEN} --influx_tag testbed ${TESTRIG} --set DUT_NAME linksys-8450 --local_lf_report_dir \
+${LOCALDIR} --enable FALSE --pull_report
+
+./ghost_profile.py --ghost_token ${GHOSTTOKEN} --ghost_host 192.168.100.153 --authors Matthew --customer candela \
+--user_push lanforge --password_push lanforge --kpi_to_ghost --grafana_token ${GRAFANATOKEN} \
+--grafana_host 192.168.100.201 --grafana_bucket lanforge_qa_testing --influx_host ${INFLUXHOST} --influx_org Candela \
+--influx_token ${INFLUXTOKEN} --influx_bucket stidmatt --parent_folder ${LOCALDIR}