Updating to make better Ghost posts, with more details in the first table, and enabling reading test_tags from the kpi.csv files.

Signed-off-by: Matthew Stidham <stidmatt@gmail.com>

py-dashboard/GhostRequest.py

@@ -194,6 +194,7 @@ class GhostRequest:
 
     def upload_image(self,
                      image):
+        if self.debug:
             print(image)
         ghost_json_url = self.ghost_json_url + '/admin/images/upload/'
 
@@ -202,6 +203,7 @@ class GhostRequest:
 
         proc = subprocess.Popen(bashCommand, shell=True, stdout=subprocess.PIPE)
         output = proc.stdout.read().decode('utf-8')
+        if self.debug:
             print(output)
         self.images.append(json.loads(output)['images'][0]['url'])
 
@@ -211,6 +213,7 @@ class GhostRequest:
             if 'kpi' in image:
                 if 'png' in image:
                     self.upload_image(folder + '/' + image)
+        if self.debug:
             print('images %s' % self.images)
 
     def custom_post(self,
@@ -274,8 +277,9 @@ class GhostRequest:
         scp_push = SCPClient(ssh_push.get_transport())
 
         if parent_folder is not None:
-            print("parent_folder %s" % parent_folder)
             files = os.listdir(parent_folder)
+            if self.debug:
+                print("parent_folder %s" % parent_folder)
                 print(files)
             for file in files:
                 if os.path.isdir(parent_folder + '/' + file) is True:
@@ -283,6 +287,7 @@ class GhostRequest:
                         shutil.rmtree(file)
                     shutil.copytree(parent_folder + '/' + file, file)
                     target_folders.append(file)
+            if self.debug:
                 print('Target folders: %s' % target_folders)
         else:
             for folder in folders:
@@ -297,6 +302,10 @@ class GhostRequest:
         images = list()
         times = list()
         test_pass_fail = list()
+        subtest_pass_fail = list()
+        subtest_pass_total = 0
+        subtest_fail_total = 0
+        test_tag = dict()
 
         for target_folder in target_folders:
             try:
@@ -304,13 +313,24 @@ class GhostRequest:
                 df = csvreader.read_csv(file=target_file, sep='\t')
                 test_rig = csvreader.get_column(df, 'test-rig')[0]
                 test_id = csvreader.get_column(df, 'test-id')[0]
+                test_tag[test_id] = (csvreader.get_column(df, 'test-tag')[0])
                 pass_fail = Counter(csvreader.get_column(df, 'pass/fail'))
                 test_pass_fail.append(pass_fail)
                 dut_hw = csvreader.get_column(df, 'dut-hw-version')[0]
                 dut_sw = csvreader.get_column(df, 'dut-sw-version')[0]
                 dut_model = csvreader.get_column(df, 'dut-model-num')[0]
                 dut_serial = csvreader.get_column(df, 'dut-serial-num')[0]
-                duts = [dut_serial, dut_hw, dut_sw, dut_model, test_rig]
+                subtest_pass = csvreader.get_column(df, 'Subtest-Pass')
+                subtest_fail = csvreader.get_column(df, 'Subtest-Fail')
+                for result in subtest_pass:
+                    subtest_pass_total += int(result)
+                for result in subtest_fail:
+                    subtest_fail_total += int(result)
+                subtest_pass_fail_list = dict()
+                subtest_pass_fail_list['PASS'] = subtest_pass_total
+                subtest_pass_fail_list['FAIL'] = subtest_fail_total
+                subtest_pass_fail.append(subtest_pass_fail_list)
+                duts = [dut_serial, dut_hw, dut_sw, dut_model, test_rig, test_tag]
                 times_append = csvreader.get_column(df, 'Date')
                 for target_time in times_append:
                     times.append(float(target_time) / 1000)
@@ -398,12 +418,13 @@ class GhostRequest:
 
 
         test_pass_fail_results = sum((Counter(test) for test in test_pass_fail), Counter())
+        subtest_pass_fail_results = sum((Counter(test) for test in subtest_pass_fail), Counter())
 
         if self.debug:
             print(times)
         end_time = max(times)
         start_time = '2021-07-01'
-        end_time = datetime.utcfromtimestamp(end_time)  # .strftime('%Y-%m-%d %H:%M:%S')
+        end_time = datetime.utcfromtimestamp(end_time)
         now = time.time()
         offset = datetime.fromtimestamp(now) - datetime.utcfromtimestamp(now)
         end_time = end_time + offset
@@ -417,6 +438,8 @@ class GhostRequest:
                                              ['Short Description', 'Score', 'Test Details'])
         high_priority.append(['Total Passed', test_pass_fail_results['PASS'], 'Total subtests passed during this run'])
         high_priority.append(['Total Failed', test_pass_fail_results['FAIL'], 'Total subtests failed during this run'])
+        high_priority.append(['Subtests Passed', subtest_pass_fail_results['PASS'], 'Total subtests passed during this run'])
+        high_priority.append(['Subtests Failed', subtest_pass_fail_results['FAIL'], 'Total subtests failed during this run'])
 
         if title is None:
             title = end_time.strftime('%B %d, %Y %I:%M %p report')
@@ -434,7 +457,8 @@ class GhostRequest:
                                         from_date=start_time,
                                         to_date=end_time.strftime('%Y-%m-%d %H:%M:%S'),
                                         pass_fail='GhostRequest',
-                                        testbed=testbeds[0])
+                                        testbed=testbeds[0],
+                                        test_tag=test_tag)
 
         if self.influx_token is not None:
             influxdb = RecordInflux(_influx_host=self.influx_host,
@@ -442,7 +466,7 @@ class GhostRequest:
                                     _influx_org=self.influx_org,
                                     _influx_token=self.influx_token,
                                     _influx_bucket=self.influx_bucket)
-            short_description = 'Ghost Post Tests passed'  # variable name
+            short_description = 'Tests passed'  # variable name
             numeric_score = test_pass_fail_results['PASS']  # value
             tags = dict()
             print(datetime.utcfromtimestamp(max(times)))
@@ -452,7 +476,7 @@ class GhostRequest:
             date = datetime.utcfromtimestamp(max(times)).isoformat()
             influxdb.post_to_influx(short_description, numeric_score, tags, date)
 
-            short_description = 'Ghost Post Tests failed'  # variable name
+            short_description = 'Tests failed'  # variable name
             numeric_score = test_pass_fail_results['FAIL']  # value
             tags = dict()
             tags['testbed'] = testbeds[0]
@@ -461,12 +485,38 @@ class GhostRequest:
             date = datetime.utcfromtimestamp(max(times)).isoformat()
             influxdb.post_to_influx(short_description, numeric_score, tags, date)
 
+            short_description = 'Subtests passed'  # variable name
+            numeric_score = subtest_pass_fail_results['PASS']  # value
+            tags = dict()
+            print(datetime.utcfromtimestamp(max(times)))
+            tags['testbed'] = testbeds[0]
+            tags['script'] = 'GhostRequest'
+            tags['Graph-Group'] = 'Subtest PASS'
+            date = datetime.utcfromtimestamp(max(times)).isoformat()
+            influxdb.post_to_influx(short_description, numeric_score, tags, date)
+
+            short_description = 'Subtests failed'  # variable name
+            numeric_score = subtest_pass_fail_results['FAIL']  # value
+            tags = dict()
+            tags['testbed'] = testbeds[0]
+            tags['script'] = 'GhostRequest'
+            tags['Graph-Group'] = 'Subtest FAIL'
+            date = datetime.utcfromtimestamp(max(times)).isoformat()
+            influxdb.post_to_influx(short_description, numeric_score, tags, date)
+
         text = 'Testbed: %s<br />' % testbeds[0]
+        test_tag_table = ''
+        for tag in list(set(test_tag.values())):
+            print(tag)
+            test_tag_table += (
+                '<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Test Tag</td>' \
+                '<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' % tag)
         dut_table = '<table width="700px" border="1" cellpadding="2" cellspacing="0" ' \
                     'style="border-color: gray; border-style: solid; border-width: 1px; "><tbody>' \
                     '<tr><th colspan="2">Test Information</th></tr>' \
                     '<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Testbed</td>' \
                     '<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
+                    '%s' \
                     '<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">DUT_HW</td>' \
                     '<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
                     '<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">DUT_SW</td>' \
@@ -478,9 +528,13 @@ class GhostRequest:
                     '<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Tests passed</td>' \
                     '<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
                     '<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Tests failed</td>' \
+                    '<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
+                    '<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Subtests passed</td>' \
+                    '<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' \
+                    '<tr><td style="border-color: gray; border-style: solid; border-width: 1px; ">Subtests failed</td>' \
                     '<td colspan="3" style="border-color: gray; border-style: solid; border-width: 1px; ">%s</td></tr>' % (
-                        duts[4], duts[1], duts[2], duts[3], duts[0], test_pass_fail_results['PASS'],
+                        duts[4], test_tag_table, duts[1], duts[2], duts[3], duts[0], test_pass_fail_results['PASS'],
-                        test_pass_fail_results['FAIL'])
+                        test_pass_fail_results['FAIL'], subtest_pass_total, subtest_fail_total)
 
         dut_table = dut_table + '</tbody></table>'
         text = text + dut_table

py-dashboard/GrafanaRequest.py

@@ -137,6 +137,17 @@ class GrafanaRequest:
                     self.units[script] = dict()
                     for index in range(0, len(graph_groups)):
                         self.units[script][graph_groups[index]] = units[index]
+                subtests = 0
+                for score in list(self.csvreader.get_column(csv, 'Subtest-Pass')):
+                    subtests += int(score)
+                for score in list(self.csvreader.get_column(csv, 'Subtest-Fail')):
+                    subtests += int(score)
+                if subtests > 0:
+                    dictionary[script].append('Subtests passed')
+                    dictionary[script].append('Subtests failed')
+                print(subtests)
+                for item in dictionary[script]:
+                    print('%s, %s' % (item, type(item)))
         print(dictionary)
         return dictionary
 
@@ -146,7 +157,8 @@ class GrafanaRequest:
                     groupBy,
                     index,
                     graph_group,
-                    testbed):
+                    testbed,
+                    test_tag=None):
         query = (
                 'from(bucket: "%s")\n  '
                 '|> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  '
@@ -158,6 +170,9 @@ class GrafanaRequest:
         if graph_group is not None:
             graphgroup = ('|> filter(fn: (r) => r["Graph-Group"] == "%s")\n' % graph_group)
             query += graphgroup
+        if test_tag is not None:
+            graphgroup = ('|> filter(fn: (r) => r["Test-Tag"] == "%s")\n' % test_tag)
+            query += graphgroup
         if testbed is not None:
             query += ('|> filter(fn: (r) => r["testbed"] == "%s")\n' % testbed)
         targets = dict()
@@ -195,7 +210,8 @@ class GrafanaRequest:
                                 to_date='now',
                                 graph_height=8,
                                 graph__width=12,
-                                pass_fail=None):
+                                pass_fail=None,
+                                test_tag=None):
         options = string.ascii_lowercase + string.ascii_uppercase + string.digits
         uid = ''.join(random.choice(options) for i in range(9))
         input1 = dict()
@@ -261,6 +277,15 @@ class GrafanaRequest:
 
                 targets = list()
                 counter = 0
+                try:
+                    new_target = self.maketargets(bucket,
+                                                  scriptname,
+                                                  groupBy,
+                                                  counter,
+                                                  graph_group,
+                                                  testbed,
+                                                  test_tag=test_tag[scriptname])
+                except:
                     new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group, testbed)
                 targets.append(new_target)
 
@@ -331,10 +356,7 @@ class GrafanaRequest:
                     panel['title'] = scriptname + ' ' + graph_group
                 else:
                     panel['title'] = scriptname
-                if 'PASS' in panel['title']:
+                print(panel['title'])
-                    panel['title'] = 'Total Passed'
-                if 'FAIL' in panel['title']:
-                    panel['title'] = 'Total Failed'
                 panel['transformations'] = list()
                 panel['transformations'].append(transformation)
                 panel['type'] = "graph"

py-json/cv_test_manager.py

@@ -20,6 +20,9 @@ def cv_base_adjust_parser(args):
         # TODO:  In future, can use TestRig once that GUI update has propagated
         args.set.append(["Test Rig ID:", args.test_rig])
 
+    if args.test_tag != "":
+        args.set.append(["TestTag", args.test_tag])
+
     if args.influx_host is not None:
         if not args.pull_report:
             print("Specified influx host without pull_report, will enabled pull_request.")
@@ -60,6 +63,8 @@ def cv_add_base_parser(parser):
     # Reporting info
     parser.add_argument("--test_rig", default="",
                         help="Specify the test rig info for reporting purposes, for instance:  testbed-01")
+    parser.add_argument("--test_tag", default="",
+                        help="Specify the test tag info for reporting purposes, for instance:  testbed-01")
 
     influx_add_parser_args(parser)  # csv_to_influx
 

py-scripts/csv_to_influx.py

@@ -61,9 +61,9 @@ class CSVtoInflux():
         columns = dict(zip(df[0], length))
         print('columns: %s' % columns)
         influx_variables = ['script', 'short-description', 'test_details', 'Graph-Group',
-                            'DUT-HW-version', 'DUT-SW-version', 'DUT-Serial-Num', 'testbed', 'Units']
+                            'DUT-HW-version', 'DUT-SW-version', 'DUT-Serial-Num', 'testbed', 'Test Tag', 'Units']
         csv_variables = ['test-id', 'short-description', 'test details', 'Graph-Group',
-                         'dut-hw-version', 'dut-sw-version', 'dut-serial-num', 'test-rig', 'Units']
+                         'dut-hw-version', 'dut-sw-version', 'dut-serial-num', 'test-rig', 'test-tag', 'Units']
         csv_vs_influx = dict(zip(csv_variables, influx_variables))
         for row in df[1:]:
             row = [sub.replace('NaN', '0') for sub in row]

py-scripts/influxgrafanaghost.sh

@@ -9,6 +9,7 @@ GHOSTTOKEN=60df4b0175953f400cd30650:d50e1fabf9a9b5d3d30fe97bc3bf04971d05496a89e9
 
 INFLUXTOKEN=31N9QDhjJHBu4eMUlMBwbK3sOjXLRAhZuCzZGeO8WVCj-xvR8gZWWvRHOcuw-5RHeB7xBFnLs7ZV023k4koR1A==
 INFLUXHOST=c7-grafana.candelatech.com
+INFLUXBUCKET=stidmatt
 
 GRAFANATOKEN=eyJrIjoiS1NGRU8xcTVBQW9lUmlTM2dNRFpqNjFqV05MZkM0dzciLCJuIjoibWF0dGhldyIsImlkIjoxfQ==
 
@@ -20,27 +21,41 @@ mkdir ${LOCALDIR}
 
 sleep 10s
 
-./create_l3.py --mgr ${MGR} --num_stations 4 --ssid stidmatt --password stidmatt --security wpa2 --radio wiphy0
+./create_l3.py --mgr ${MGR} --num_stations 4 --ssid stidmatt2 --password stidmatt2 --security wpa2 --radio wiphy0
 
 ./lf_dataplane_test.py --mgr ${MGR} --lf_user ${LFUSER} --lf_password lanforge --instance_name wct_instance \
 --config_name 64_stations --upstream 1.1.eth1 --influx_host c7-grafana.candelatech.com --influx_org Candela \
---influx_token ${INFLUXTOKEN} --influx_bucket stidmatt --test_rig {TESTRIG} --influx_tag testbed ${TESTRIG} \
+--influx_token ${INFLUXTOKEN} --influx_bucket ${INFLUXBUCKET} --test_rig ${TESTRIG} --influx_tag testbed ${TESTRIG} \
 --station 1.1.sta0000 --set DUT_NAME linksys-8450 --local_lf_report_dir ${LOCALDIR} \
 --pull_report \
 --download_speed 85% --upload_speed 0 \
 --raw_line 'cust_pkt_sz: 88 1200' \
 --raw_line 'directions: DUT Transmit;DUT Receive' \
---raw_line 'traffic_types: UDP' \
+--raw_line 'traffic_types: UDP' --pull_report --test_tag influxgrafanaghost.sh
---test_rig Testbed-01 --pull_report
 #--raw_line 'pkts: Custom;60;142;256;512;1024;MTU'
 
 ./lf_wifi_capacity_test.py --mgr ${MGR} --lf_user ${LFUSER} --lf_password lanforge --instance_name linksys-8450 \
 --config_name wifi_config --upstream 1.1.eth1 --radio wiphy0 --ssid lanforge --paswd lanforge --security wpa2 \
---influx_host ${INFLUXHOST} --influx_org Candela --influx_bucket stidmatt --test_rig ${TESTRIG} \
+--influx_host ${INFLUXHOST} --influx_org Candela --influx_bucket ${INFLUXBUCKET} --test_rig ${TESTRIG} \
 --influx_token ${INFLUXTOKEN} --influx_tag testbed ${TESTRIG} --set DUT_NAME linksys-8450 --local_lf_report_dir \
-${LOCALDIR} --enable FALSE --pull_report
+${LOCALDIR} --enable FALSE --pull_report --test_tag influxgrafanaghost.sh
+
+./lf_wifi_capacity_test.py --mgr ${MGR} --lf_user ${LFUSER} --lf_password lanforge --instance_name linksys-8450 \
+--config_name wifi_config --upstream 1.1.eth1 --radio wiphy0 --ssid lanforge --paswd lanforge --security wpa2 \
+--influx_host ${INFLUXHOST} --influx_org Candela --influx_bucket ${INFLUXBUCKET} --test_rig ${TESTRIG} \
+--influx_token ${INFLUXTOKEN} --influx_tag testbed ${TESTRIG} --set DUT_NAME linksys-8450 --local_lf_report_dir \
+${LOCALDIR} --enable FALSE --pull_report --test_tag Can_we_use_two_test_tags
+
+./lf_ap_auto_test.py --mgr ${MGR} --instance_name ap-auto-instance --config_name test_con --upstream 1.1.eth1 \
+--dut5_0 'matthew-router lanforge 04:f0:21:c0:65:7b (1)' --dut2_0 'matthew-router lanforge 04:f0:21:c0:65:7b (1)' \
+--max_stations_2 32 --max_stations_5 32 --max_stations_dual 100 --radio2 1.1.wiphy0 --radio5 1.1.wiphy0 \
+--set 'Basic Client Connectivity' 1 --set 'Multi Band Performance' 1 --set 'Stability' 0   --set 'Capacity' 0 \
+--set 'Multi-Station Throughput vs Pkt Size' 0 --set 'Throughput vs Pkt Size' 0 --set 'Band-Steering' 1 \
+--influx_host ${INFLUXHOST} --influx_org Candela --influx_bucket ${INFLUXBUCKET} --test_rig ${TESTRIG} \
+--influx_token ${INFLUXTOKEN} --influx_tag testbed ${TESTRIG} --pull_report --test_tag influxgrafanaghost.sh \
+--local_lf_report_dir ${LOCALDIR}
 
 ./ghost_profile.py --ghost_token ${GHOSTTOKEN} --ghost_host 192.168.100.153 --authors Matthew --customer candela \
 --user_push lanforge --password_push lanforge --kpi_to_ghost --grafana_token ${GRAFANATOKEN} \
---grafana_host 192.168.100.201 --grafana_bucket lanforge_qa_testing --influx_host ${INFLUXHOST} --influx_org Candela \
+--grafana_host 192.168.100.201 --grafana_bucket ${INFLUXBUCKET} --influx_host ${INFLUXHOST} --influx_org Candela \
---influx_token ${INFLUXTOKEN} --influx_bucket stidmatt --parent_folder ${LOCALDIR}
+--influx_token ${INFLUXTOKEN} --influx_bucket ${INFLUXBUCKET} --parent_folder ${LOCALDIR}

py-scripts/lf_wifi_capacity_test.py

@@ -318,7 +318,6 @@ if sys.version_info[0] != 3:
 if 'py-json' not in sys.path:
     sys.path.append(os.path.join(os.path.abspath('..'), 'py-json'))
 
-from cv_test_manager import cv_test
 from cv_test_manager import *
 from LANforge import LFUtils
 
@@ -357,6 +356,7 @@ class WiFiCapacityTest(cv_test):
                  report_dir="",
                  graph_groups=None,
                  test_rig="",
+                 test_tag="",
                  local_lf_report_dir=""
                  ):
         super().__init__(lfclient_host=lfclient_host, lfclient_port=lf_port)
@@ -395,6 +395,7 @@ class WiFiCapacityTest(cv_test):
         self.report_dir = report_dir
         self.graph_groups = graph_groups
         self.test_rig = test_rig
+        self.test_tag = test_tag
         self.local_lf_report_dir = local_lf_report_dir
 
     def setup(self):
@@ -453,6 +454,8 @@ class WiFiCapacityTest(cv_test):
             cfg_options.append("dl_rate: " + self.download_rate)
         if self.test_rig != "":
             cfg_options.append("test_rig: " + self.test_rig)
+        if self.test_tag != "":
+            cfg_options.append("test_tag: " + self.test_tag)
 
         cfg_options.append("save_csv: 1")
 
@@ -487,7 +490,7 @@ def main():
              --instance_name wct_instance --config_name wifi_config --upstream 1.1.eth1 --batch_size 1 --loop_iter 1 \
              --protocol UDP-IPv4 --duration 6000 --pull_report --stations 1.1.sta0000,1.1.sta0001 \
              --create_stations --radio wiphy0 --ssid test-ssid --security open --paswd [BLANK] \
-             --test_rig Testbed-01 \
+             --test_rig Testbed-01 -test_tag TAG\
              --influx_host c7-graphana --influx_port 8086 --influx_org Candela \
              --influx_token=-u_Wd-L8o992701QF0c5UmqEp7w7Z7YOMaWLxOMgmHfATJGnQbbmYyNxHBR9PgD6taM_tcxqJl6U8DjU1xINFQ== \
              --influx_bucket ben \
@@ -562,6 +565,7 @@ def main():
                                 sets=args.set,
                                 graph_groups=args.graph_groups,
                                 test_rig=args.test_rig,
+                                test_tag=args.test_tag,
                                 local_lf_report_dir=args.local_lf_report_dir
                                 )
     WFC_Test.setup()

py-scripts/regression_test.sh

@@ -131,7 +131,7 @@ if [[ $MGRLEN -gt 0 ]]; then
       testgroup_list_groups
       testgroup_list_connections
       testgroup_delete_group
-      "./testgroup2.py --num_stations 4 --ssid lanforge --passwd password --security wpa2 --radio wiphy0 --group_name group0 --add_group --mgr $MGR"
+      "./testgroup2.py --num_stations 4 --ssid $SSID_USED --passwd $PASSWD_USED --security $SECURITY --radio $RADIO_USED --group_name group0 --add_group --mgr $MGR"
       "./test_ipv4_connection.py --radio $RADIO_USED --num_stations $NUM_STA --ssid $SSID_USED --passwd $PASSWD_USED --security $SECURITY --debug --mgr $MGR"
       "./test_ipv4_l4_urls_per_ten.py --radio $RADIO_USED --num_stations $NUM_STA --security $SECURITY --ssid $SSID_USED --passwd $PASSWD_USED --num_tests 1 --requests_per_ten 600 --target_per_ten 600 --debug --mgr $MGR"
       "./test_ipv4_l4_wifi.py --radio $RADIO_USED --num_stations $NUM_STA --security $SECURITY --ssid $SSID_USED --passwd $PASSWD_USED --test_duration 15s --debug --mgr $MGR"