Add failure message when a KPI is not found in a folder for GhostRequest

Automatic script to run influx, grafana, and ghost from wifi capacity and dataplane.

Minor fix in csv_to_influx for correcting missing values

Signed-off-by: Matthew Stidham <stidmatt@gmail.com>

py-dashboard/GhostRequest.py

@@ -291,8 +291,7 @@ class GhostRequest:
                 target_folders.append(folder)
 
         testbeds = list()
-        webpages = self.webpages
-        pdfs = self.pdfs
+        webpagesandpdfs = list()
         high_priority_list = list()
         low_priority_list = list()
         images = list()
@@ -304,6 +303,7 @@ class GhostRequest:
                 target_file = '%s/kpi.csv' % target_folder
                 df = csvreader.read_csv(file=target_file, sep='\t')
                 test_rig = csvreader.get_column(df, 'test-rig')[0]
+                test_id = csvreader.get_column(df, 'test-id')[0]
                 pass_fail = Counter(csvreader.get_column(df, 'pass/fail'))
                 test_pass_fail.append(pass_fail)
                 dut_hw = csvreader.get_column(df, 'dut-hw-version')[0]
@@ -323,66 +323,78 @@ class GhostRequest:
                     text = text + 'Tests passed: 0<br />' \
                                   'Tests failed : 0<br />' \
                                   'Percentage of tests passed: Not Applicable<br />'
+                testbeds.append(test_rig)
+                if testbed is None:
+                    testbed = test_rig
+
+                if test_run is None:
+                    test_run = now.strftime('%B-%d-%Y-%I-%M-%p-report')
+
+                local_path = '/home/%s/%s/%s/%s' % (user_push, customer, testbed, test_run)
+
+                transport = paramiko.Transport(ghost_host, port)
+                transport.connect(None, user_push, password_push)
+                sftp = paramiko.sftp_client.SFTPClient.from_transport(transport)
+
+                if self.debug:
+                    print(local_path)
+                    print(target_folder)
+
+                try:
+                    sftp.mkdir('/home/%s/%s/%s' % (user_push, customer, testbed))
+                except:
+                    pass
+
+                try:
+                    sftp.mkdir(local_path)
+                except:
+                    pass
+                scp_push.put(target_folder, local_path, recursive=True)
+                files = sftp.listdir(local_path + '/' + target_folder)
+                pdfs = list()
+                webpages = list()
+                for file in files:
+                    if 'pdf' in file:
+                        url = 'http://%s/%s/%s/%s/%s/%s' % (
+                            ghost_host, customer.strip('/'), testbed, test_run, target_folder, file)
+                        pdfs.append('<a href="%s">PDF</a>' % url)
+                if 'index.html' in files:
+                    url = 'http://%s/%s/%s/%s/%s/%s' % (
+                        ghost_host, customer.strip('/'), testbed, test_run, target_folder, 'index.html')
+                    webpages.append('<a href="%s">HTML</a>' % url)
+                webpagesandpdfsappend = dict()
+                webpagesandpdfsappend[test_id] = pdfs + webpages
+                webpagesandpdfs.append(webpagesandpdfsappend)
+                scp_push.close()
+                self.upload_images(target_folder)
+                for image in self.images:
+                    if 'kpi-' in image:
+                        if '-print' not in image:
+                            images.append('<img src="%s"></img>' % image)
+                self.images = []
+
+                results = csvreader.get_columns(df, ['short-description', 'numeric-score', 'test details', 'pass/fail',
+                                                     'test-priority'])
+
+                results[0] = ['Short Description', 'Score', 'Test Details', 'Pass or Fail', 'test-priority']
+                for row in results:
+                    try:
+                        row[1] = round(float(row[1]), 2)
+                    except:
+                        pass
+
+                low_priority = csvreader.filter_df(results, 'test-priority', 'less than', 94)
+                high_priority = csvreader.filter_df(results, 'test-priority', 'greater than or equal to', 95)
+                high_priority_list.append(high_priority)
+
+                low_priority_list.append(low_priority)
 
             except:
                 print("Failure")
                 target_folders.remove(target_folder)
-                break
-            testbeds.append(test_rig)
-            if testbed is None:
-                testbed = test_rig
-
-            if test_run is None:
-                test_run = now.strftime('%B-%d-%Y-%I-%M-%p-report')
-
-            local_path = '/home/%s/%s/%s/%s' % (user_push, customer, testbed, test_run)
-
-            transport = paramiko.Transport(ghost_host, port)
-            transport.connect(None, user_push, password_push)
-            sftp = paramiko.sftp_client.SFTPClient.from_transport(transport)
-
-            if self.debug:
-                print(local_path)
-                print(target_folder)
-
-            try:
-                sftp.mkdir('/home/%s/%s/%s' % (user_push, customer, testbed))
-            except:
-                pass
-
-            try:
-                sftp.mkdir(local_path)
-            except:
-                pass
-            scp_push.put(target_folder, local_path, recursive=True)
-            files = sftp.listdir(local_path + '/' + target_folder)
-            for file in files:
-                if 'pdf' in file:
-                    url = 'http://%s/%s/%s/%s/%s/%s' % (
-                        ghost_host, customer.strip('/'), testbed, test_run, target_folder, file)
-                    pdfs.append('PDF of results: <a href="%s">%s</a>' % (url, file))
-            if 'index.html' in files:
-                url = 'http://%s/%s/%s/%s/%s/%s' % (
-                    ghost_host, customer.strip('/'), testbed, test_run, target_folder, 'index.html')
-                webpages.append('Results webpage: <a href="%s">%s</a><br />' % (url, target_folder))
-            scp_push.close()
-            self.upload_images(target_folder)
-            for image in self.images:
-                if 'kpi-' in image:
-                    if '-print' not in image:
-                        images.append('<img src="%s"></img>' % image)
-            self.images = []
-
-            results = csvreader.get_columns(df, ['short-description', 'numeric-score', 'test details', 'pass/fail',
-                                                 'test-priority'])
-
-            results[0] = ['Short Description', 'Score', 'Test Details', 'Pass or Fail', 'test-priority']
-
-            low_priority = csvreader.filter_df(results, 'test-priority', 'less than', 94)
-            high_priority = csvreader.filter_df(results, 'test-priority', 'greater than or equal to', 95)
-            high_priority_list.append(high_priority)
-
-            low_priority_list.append(low_priority)
+                failuredict = dict()
+                failuredict[target_folder] = ['Failure']
+                webpagesandpdfs.append(failuredict)
 
 
         test_pass_fail_results = sum((Counter(test) for test in test_pass_fail), Counter())
@@ -391,7 +403,7 @@ class GhostRequest:
             print(times)
         end_time = max(times)
         start_time = '2021-07-01'
-        end_time = datetime.utcfromtimestamp(end_time)#.strftime('%Y-%m-%d %H:%M:%S')
+        end_time = datetime.utcfromtimestamp(end_time)  # .strftime('%Y-%m-%d %H:%M:%S')
         now = time.time()
         offset = datetime.fromtimestamp(now) - datetime.utcfromtimestamp(now)
         end_time = end_time + offset
@@ -473,10 +485,14 @@ class GhostRequest:
         dut_table = dut_table + '</tbody></table>'
         text = text + dut_table
 
-        for article in zip(pdfs, webpages):
-            if self.debug:
-                print(article)
-            text = text + article[0] + ' | ' + article[1]
+        for dictionary in webpagesandpdfs:
+            text += list(dictionary.keys())[0] + ' report: '
+            for value in dictionary.values():
+                for webpage in value:
+                    text += webpage
+                    if value.index(webpage) + 1 != len(value):
+                        text += ' | '
+            text += '<br />'
 
         for image in images:
             text = text + image

py-scripts/csv_to_influx.py

@@ -57,8 +57,6 @@ class CSVtoInflux():
     # Submit data to the influx db if configured to do so.
     def post_to_influx(self):
         df = self.read_csv(self.target_csv)
-        for row in df:
-            row =[sub.replace('NaN','0') for sub in row]
         length = list(range(0, len(df[0])))
         columns = dict(zip(df[0], length))
         print('columns: %s' % columns)
@@ -68,6 +66,7 @@ class CSVtoInflux():
                          'dut-hw-version', 'dut-sw-version', 'dut-serial-num', 'test-rig', 'Units']
         csv_vs_influx = dict(zip(csv_variables, influx_variables))
         for row in df[1:]:
+            row = [sub.replace('NaN', '0') for sub in row]
             tags = dict()
             print("row: %s" % row)
             short_description = row[columns['short-description']]

py-scripts/influxgrafanaghost.sh

@@ -0,0 +1,46 @@
+#! /bin/bash
+
+MGR=192.168.100.213
+LFUSER=lanforge
+LOCALDIR=/home/matthew/Documents/lanforge-scripts/py-scripts/lftest
+TESTRIG="Matthew-ct523c"
+
+GHOSTTOKEN=60df4b0175953f400cd30650:d50e1fabf9a9b5d3d30fe97bc3bf04971d05496a89e92a169a0d72357c81f742
+
+INFLUXTOKEN=31N9QDhjJHBu4eMUlMBwbK3sOjXLRAhZuCzZGeO8WVCj-xvR8gZWWvRHOcuw-5RHeB7xBFnLs7ZV023k4koR1A==
+INFLUXHOST=c7-grafana.candelatech.com
+
+GRAFANATOKEN=eyJrIjoiS1NGRU8xcTVBQW9lUmlTM2dNRFpqNjFqV05MZkM0dzciLCJuIjoibWF0dGhldyIsImlkIjoxfQ==
+
+rm -r ${LOCALDIR}
+
+mkdir ${LOCALDIR}
+
+./scenario.py --mgr ${MGR} --load BLANK
+
+sleep 10s
+
+./create_l3.py --mgr ${MGR} --num_stations 4 --ssid stidmatt --password stidmatt --security wpa2 --radio wiphy0
+
+./lf_dataplane_test.py --mgr ${MGR} --lf_user ${LFUSER} --lf_password lanforge --instance_name wct_instance \
+--config_name 64_stations --upstream 1.1.eth1 --influx_host c7-grafana.candelatech.com --influx_org Candela \
+--influx_token ${INFLUXTOKEN} --influx_bucket stidmatt --test_rig {TESTRIG} --influx_tag testbed ${TESTRIG} \
+--station 1.1.sta0000 --set DUT_NAME linksys-8450 --local_lf_report_dir ${LOCALDIR} \
+--pull_report \
+--download_speed 85% --upload_speed 0 \
+--raw_line 'cust_pkt_sz: 88 1200' \
+--raw_line 'directions: DUT Transmit;DUT Receive' \
+--raw_line 'traffic_types: UDP' \
+--test_rig Testbed-01 --pull_report
+#--raw_line 'pkts: Custom;60;142;256;512;1024;MTU'
+
+./lf_wifi_capacity_test.py --mgr ${MGR} --lf_user ${LFUSER} --lf_password lanforge --instance_name linksys-8450 \
+--config_name wifi_config --upstream 1.1.eth1 --radio wiphy0 --ssid lanforge --paswd lanforge --security wpa2 \
+--influx_host ${INFLUXHOST} --influx_org Candela --influx_bucket stidmatt --test_rig ${TESTRIG} \
+--influx_token ${INFLUXTOKEN} --influx_tag testbed ${TESTRIG} --set DUT_NAME linksys-8450 --local_lf_report_dir \
+${LOCALDIR} --enable FALSE --pull_report
+
+./ghost_profile.py --ghost_token ${GHOSTTOKEN} --ghost_host 192.168.100.153 --authors Matthew --customer candela \
+--user_push lanforge --password_push lanforge --kpi_to_ghost --grafana_token ${GRAFANATOKEN} \
+--grafana_host 192.168.100.201 --grafana_bucket lanforge_qa_testing --influx_host ${INFLUXHOST} --influx_org Candela \
+--influx_token ${INFLUXTOKEN} --influx_bucket stidmatt --parent_folder ${LOCALDIR}