Updates to import statements:

- Import importlib, os, and sys to each python script.
 - Append "lanforge-scripts" root directory to the system path, allowing each script to be called from an antecedent directory. e.g.
if 'lanforge-scripts' not in sys.path:
    sys.path.append(os.path.join(os.path.abspath(__file__ + "../../../../../")))
 - All statements of the form 'from <module> import <class>' replace with:
<module> = importlib.import_module("lanforge-scripts.<directory>.<module>")
<class> = <module>.<class>
This commit is contained in:
erinnerim
2021-09-15 16:17:16 -07:00
parent 5404d9068b
commit 1d7578251c
128 changed files with 1639 additions and 1388 deletions

View File

@@ -44,34 +44,38 @@ COPYRIGHT:
Copyright 2021 Candela Technologies Inc
INCLUDE_IN_README
'''
import sys
import os
import importlib
from pprint import pprint
import re
import serial
import pexpect
from pexpect_serial import SerialSpawn
import argparse
import time
import datetime
import subprocess
import csv
if sys.version_info[0] != 3:
print("This script requires Python 3")
exit(1)
if 'py-json' not in sys.path:
sys.path.append(os.path.join(os.path.abspath('..'), 'py-json'))
if 'py-dashboard' not in sys.path:
sys.path.append(os.path.join(os.path.abspath('..'), 'py-dashboard'))
if 'lanforge-scripts' not in sys.path:
sys.path.append(os.path.join(os.path.abspath(__file__ + "../../../../")))
lf_report = importlib.import_module("lanforge-scripts.py-scripts.lf_report")
# lfcli_base = importlib.import_module("lanforge-scripts.py-json.LANforge.lfcli_base")
# LFCliBase = lfcli_base.LFCliBase
LFUtils = importlib.import_module("lanforge-scripts.py-json.LANforge.LFUtils")
realm = importlib.import_module("lanforge-scripts.py-json.realm")
Realm = realm.Realm
csv_to_influx = importlib.import_module("lanforge-scripts.py-scripts.csv_to_influx")
InfluxRequest = importlib.import_module("lanforge-scripts.py-dashboard.InfluxRequest")
influx_add_parser_args = InfluxRequest.influx_add_parser_args
from pprint import pprint
from InfluxRequest import *
import serial
import pexpect
from pexpect_serial import SerialSpawn
from lf_report import lf_report
import argparse
from LANforge import LFUtils
from realm import Realm
import time
import datetime
import csv
# This class handles running the test and generating reports.
class L3VariableTime(Realm):
@@ -288,83 +292,52 @@ class L3VariableTime(Realm):
def get_endp_stats_for_port(self, eid_name, endps):
lat = 0
jit = 0
total_dl_rate = 0
total_dl_rate_ll = 0
total_dl_pkts_ll = 0
total_ul_rate = 0
total_ul_rate_ll = 0
total_ul_pkts_ll = 0
tput = 0
count = 0
sta_name = 'no_station'
#print("endp-stats-for-port, port-eid: {}".format(eid_name))
eid = self.name_to_eid(eid_name)
print("eid_name: {eid_name} eid: {eid}".format(eid_name=eid_name,eid=eid))
# Convert all eid elements to strings
eid[0] = str(eid[0])
eid[1] = str(eid[1])
eid[2] = str(eid[2])
for endp in endps:
pprint(endp)
eid_endp = endp["eid"].split(".")
print("Comparing eid:{eid} to endp-id {eid_endp}".format(eid=eid,eid_endp=eid_endp))
for e in endps:
#pprint(e)
eid_endp = e["eid"].split(".")
print("Comparing eid: ", eid, " to endp-id: ", eid_endp)
# Look through all the endpoints (endps), to find the port the eid_name is using.
# The eid_name that has the same Shelf, Resource, and Port as the eid_endp (looking at all the endps)
# Then read the eid_endp to get the delay, jitter and rx rate
# Note: the endp eid is shelf.resource.port.endp-id, the eid can be treated somewhat as
# child class of port-eid , and look up the port the eid is using.
if eid[0] == eid_endp[0] and eid[1] == eid_endp[1] and eid[2] == eid_endp[2]:
lat += int(endp["delay"])
jit += int(endp["jitter"])
name = endp["name"]
print("endp name {name}".format(name=name))
sta_name = name.replace('-A','')
#only the -A endpoint will be found so need to look
lat += int(e["delay"])
jit += int(e["jitter"])
tput += int(e["rx rate"])
count += 1
print("Matched: name: {name} eid:{eid} to endp-id {eid_endp}".format(name=name,eid=eid,eid_endp=eid_endp))
print("matched: ")
else:
name = endp["name"]
print("No Match: name: {name} eid:{eid} to endp-id {eid_endp}".format(name=name,eid=eid,eid_endp=eid_endp))
print("Did not match")
if count > 1:
lat = int(lat / count)
jit = int(jit / count)
# need to loop though again to find the upload and download per station if the name matched
for endp in endps:
if sta_name in endp["name"]:
name = endp["name"]
if name.endswith("-A"):
print("name has -A")
total_dl_rate += int(endp["rx rate"])
total_dl_rate_ll += int(endp["rx rate ll"])
total_dl_pkts_ll += int(endp["rx pkts ll"])
# -B upload side
else:
total_ul_rate += int(endp["rx rate"])
total_ul_rate_ll += int(endp["rx rate ll"])
total_ul_pkts_ll += int(endp["rx pkts ll"])
return lat, jit, total_dl_rate, total_dl_rate_ll, total_dl_pkts_ll, total_ul_rate, total_ul_rate_ll, total_ul_pkts_ll
return lat, jit, tput
# Query all endpoints to generate rx and other stats, returned
# as an array of objects.
def __get_rx_values(self):
endp_list = self.json_get("endp?fields=name,eid,delay,jitter,rx+rate,rx+rate+ll,rx+bytes,rx+drop+%25,rx+pkts+ll", debug_=False)
endp_list = self.json_get("endp?fields=name,eid,delay,jitter,rx+rate,rx+bytes,rx+drop+%25", debug_=False)
endp_rx_drop_map = {}
endp_rx_map = {}
our_endps = {}
endps = []
total_ul = 0
total_ul_ll = 0
total_dl = 0
total_dl_ll = 0
for e in self.multicast_profile.get_mc_names():
our_endps[e] = e;
@@ -377,16 +350,10 @@ class L3VariableTime(Realm):
endps.append(value)
print("endpoint: ", item, " value:\n")
pprint(value)
for value_name, value in value.items():
if value_name == 'rx bytes':
endp_rx_map[item] = value
if value_name == 'rx rate':
endp_rx_map[item] = value
if value_name == 'rx rate ll':
endp_rx_map[item] = value
if value_name == 'rx pkts ll':
endp_rx_map[item] = value
if value_name == 'rx drop %':
endp_rx_drop_map[item] = value
if value_name == 'rx rate':
@@ -396,16 +363,9 @@ class L3VariableTime(Realm):
total_dl += int(value)
else:
total_ul += int(value)
if value_name == 'rx rate ll':
# This hack breaks for mcast or if someone names endpoints weirdly.
if item.endswith("-A"):
total_dl_ll += int(value)
else:
total_ul_ll += int(value)
#print("total-dl: ", total_dl, " total-ul: ", total_ul, "\n")
return endp_rx_map, endp_rx_drop_map, endps, total_dl, total_ul, total_dl_ll, total_ul_ll
return endp_rx_map, endp_rx_drop_map, endps, total_dl, total_ul
# Common code to generate timestamp for CSV files.
def time_stamp(self):
@@ -535,7 +495,7 @@ class L3VariableTime(Realm):
ss.sendline(str(self.ap_cmd_5g))
ss.expect([pexpect.TIMEOUT], timeout=1) # do not detete line, waits for output
ap_stats_5g = ss.before.decode('utf-8','ignore')
print("ap_stats_5g from AP: {}".format(ap_stats_5g))
print("ap_stats_5g {}".format(ap_stats_5g))
except:
print("WARNING unable to read AP")
@@ -551,8 +511,8 @@ class L3VariableTime(Realm):
ss = SerialSpawn(ser)
ss.sendline(str(self.ap_cmd_2g))
ss.expect([pexpect.TIMEOUT], timeout=1) # do not detete line, waits for output
ap_stats_2g = ss.before.decode('utf-8','ignore')
print("ap_stats_2g from AP: {}".format(ap_stats_2g))
ap_stats = ss.before.decode('utf-8','ignore')
print("ap_stats_2g {}".format(ap_stats_2g))
except:
print("WARNING unable to read AP")
@@ -683,7 +643,7 @@ class L3VariableTime(Realm):
cur_time = datetime.datetime.now()
print("Getting initial values.")
old_rx_values, rx_drop_percent, endps, total_dl_bps, total_ul_bps, total_dl_ll_bps, total_ul_ll_bps = self.__get_rx_values()
old_rx_values, rx_drop_percent, endps, total_dl_bps, total_ul_bps = self.__get_rx_values()
end_time = self.parse_time(self.test_duration) + cur_time
@@ -694,8 +654,6 @@ class L3VariableTime(Realm):
expected_passes = 0
total_dl_bps = 0
total_ul_bps = 0
total_dl_ll_bps = 0
total_ul_ll_bps = 0
endps = []
ap_row = []
ap_stats_col_titles = []
@@ -712,9 +670,9 @@ class L3VariableTime(Realm):
time.sleep(.2)
self.epoch_time = int(time.time())
new_rx_values, rx_drop_percent, endps, total_dl_bps, total_ul_bps, total_dl_ll_bps, total_ul_ll_bps = self.__get_rx_values()
new_rx_values, rx_drop_percent, endps, total_dl_bps, total_ul_bps = self.__get_rx_values()
print("main loop, total-dl: ", total_dl_bps, " total-ul: ", total_ul_bps, " total-dl-ll: ", total_dl_ll_bps," total-ul-ll: ", total_ul_ll_bps)
#print("main loop, total-dl: ", total_dl_bps, " total-ul: ", total_ul_bps)
# AP OUTPUT
if self.ap_read:
@@ -733,9 +691,7 @@ class L3VariableTime(Realm):
ap_chanim_stats_5g = "{}{}{}{}".format("root@Docsis-Gateway:~# wl -i wl1 chanim_stats\n",
"version: 3\n",
"chanspec tx inbss obss nocat nopkt doze txop goodtx badtx glitch badplcp knoise idle timestamp\n",
#`"0xe06a 61 15 0 17 0 0 6 53 2 0 0 -91 65 343370578\n")
"0xe06a 1.67 15.00 0.00 17.00 0.00 0.00 97.33 53.00 2.00 0 0 -91 65 343370578\n")
"0xe06a 61 15 0 17 0 0 6 53 2 0 0 -91 65 343370578\n")
else:
# read from the AP
ap_stats_5g = self.read_ap_stats_5g()
@@ -763,7 +719,7 @@ class L3VariableTime(Realm):
# print("response".format(response))
pprint(response)
p = response['interface']
print("#### From LANforge: p, response['insterface']:{}".format(p))
#print("#### From LANforge: p, response['insterface']:{}".format(p))
mac = p['mac']
#print("#### From LANforge: p['mac']: {mac}".format(mac=mac))
@@ -791,7 +747,7 @@ class L3VariableTime(Realm):
print("selected ap_row (from split_row): {}".format(ap_row))
# Find latency, jitter for connections using this port.
latency, jitter, total_ul_rate, total_ul_rate_ll, total_ul_pkts_ll, total_dl_rate, total_dl_rate_ll, total_dl_pkts_ll = self.get_endp_stats_for_port(p["port"], endps)
latency, jitter, tput = self.get_endp_stats_for_port(p["port"], endps)
# now report the ap_chanim_stats along side of the ap_stats_5g
xtop_reported = False
@@ -800,9 +756,7 @@ class L3VariableTime(Realm):
if xtop_reported:
try:
xtop = split_row[7]
print("xtop 5g {xtop}".format(xtop=xtop))
channel_utilization = float(100) - float(xtop)
print("channel_utilization {utilization}".format(utilization=channel_utilization))
channel_utilization = 100 - int(xtop)
except:
print("detected chanspec with reading chanim_stats, failed reading xtop")
# should be only one channel utilization
@@ -821,8 +775,7 @@ class L3VariableTime(Realm):
ap_stats_5g_col_titles = ['Station Address','PHY Mbps','Data Mbps','Air Use','Data Use','Retries','bw','mcs','Nss','ofdma','mu-mimo','channel_utilization']
self.write_port_csv(len(temp_stations_list), ul, dl, ul_pdu_str, dl_pdu_str, atten_val, eid_name, p,
latency, jitter, total_ul_rate, total_ul_rate_ll, total_ul_pkts_ll,
total_dl_rate, total_dl_rate_ll, total_dl_pkts_ll , ap_row, ap_stats_5g_col_titles) #ap_stats_5g_col_titles used as a length
latency, jitter, tput, ap_row, ap_stats_5g_col_titles) #ap_stats_5g_col_titles used as a length
if self.ap_test_mode:
# Create the test data as a continuous string
ap_stats_2g="{}{}{}{}{}{}".format("root@Docsis-Gateway:~# wl -i wl1 bs_data\n",
@@ -838,8 +791,7 @@ class L3VariableTime(Realm):
ap_chanim_stats_2g = "{}{}{}{}".format("root@Docsis-Gateway:~# wl -i wl1 chanim_stats\n",
"version: 3\n",
"chanspec tx inbss obss nocat nopkt doze txop goodtx badtx glitch badplcp knoise idle timestamp\n",
#"0xe06a 62 15 0 17 0 0 6 53 2 0 0 -91 65 343370578\n")
"0xe06a 1.67 15.00 0.00 17.00 0.00 0.00 97.33 53.00 2.00 0 0 -91 65 343370578\n")
"0xe06a 61 15 0 17 0 0 6 53 2 0 0 -91 65 343370578\n")
else:
# read from the AP
ap_stats_2g = self.read_ap_stats_2g()
@@ -895,7 +847,7 @@ class L3VariableTime(Realm):
print("selected ap_row (from split_row): {}".format(ap_row))
# Find latency, jitter for connections using this port.
latency, jitter, total_ul_rate, total_ul_rate_ll, total_ul_pkts_ll, total_dl_rate, total_dl_rate_ll, total_dl_pkts_ll = self.get_endp_stats_for_port(p["port"], endps)
latency, jitter, tput = self.get_endp_stats_for_port(p["port"], endps)
# now report the ap_chanim_stats along side of the ap_stats_2g
xtop_reported = False
@@ -904,9 +856,7 @@ class L3VariableTime(Realm):
if xtop_reported:
try:
xtop = split_row[7]
print("xtop 2g {xtop}".format(xtop=xtop))
channel_utilization = float(100) - float(xtop)
print("channel_utilization 2g {utilization}".format(utilization=channel_utilization))
channel_utilization = 100 - int(xtop)
except:
print("detected chanspec with reading chanim_stats, failed reading xtop")
# should be only one channel utilization
@@ -925,8 +875,7 @@ class L3VariableTime(Realm):
ap_stats_2g_col_titles = ['Station Address','PHY Mbps','Data Mbps','Air Use','Data Use','Retries','bw','mcs','Nss','ofdma','mu-mimo','channel_utilization']
self.write_port_csv(len(temp_stations_list), ul, dl, ul_pdu_str, dl_pdu_str, atten_val, eid_name, p,
latency, jitter, total_ul_rate, total_ul_rate_ll, total_ul_pkts_ll,
total_dl_rate, total_dl_rate_ll, total_dl_pkts_ll, ap_row, ap_stats_2g_col_titles) #ap_stats_2g_col_titles used as a length
latency, jitter, tput, ap_row, ap_stats_2g_col_titles) #ap_stats_2g_col_titles used as a length
else:
@@ -942,15 +891,14 @@ class L3VariableTime(Realm):
pprint(response)
else:
p = response['interface']
latency, jitter, total_ul_rate, total_ul_rate_ll, total_ul_pkts_ll, total_dl_rate, total_dl_rate_ll, total_dl_pkts_ll= self.get_endp_stats_for_port(p["port"], endps)
latency, jitter, tput = self.get_endp_stats_for_port(p["port"], endps)
self.write_port_csv(len(temp_stations_list), ul, dl, ul_pdu_str, dl_pdu_str, atten_val, eid_name, p,
latency, jitter, total_ul_rate, total_ul_rate_ll, total_ul_pkts_ll,
total_dl_rate, total_dl_rate_ll, total_dl_pkts_ll, ap_row, ap_stats_col_titles) #ap_stats_col_titles used as a length
latency, jitter, tput, ap_row, ap_stats_col_titles) #ap_stats_col_titles used as a length
# At end of test step, record KPI information. This is different the kpi.csv
self.record_kpi(len(temp_stations_list), ul, dl, ul_pdu_str, dl_pdu_str, atten_val, total_dl_bps, total_ul_bps, total_dl_ll_bps, total_ul_ll_bps)
# At end of test step, record KPI information.
self.record_kpi(len(temp_stations_list), ul, dl, ul_pdu_str, dl_pdu_str, atten_val, total_dl_bps, total_ul_bps)
# At end of test if requested store upload and download stats
if self.ap_scheduler_stats:
@@ -975,15 +923,15 @@ class L3VariableTime(Realm):
if passes == expected_passes:
self._pass("PASS: Requested-Rate: %s <-> %s PDU: %s <-> %s All tests passed" % (ul, dl, ul_pdu, dl_pdu), print_pass)
def write_port_csv(self, sta_count, ul, dl, ul_pdu, dl_pdu, atten, eid_name, port_data, latency, jitter, total_ul_rate, total_ul_rate_ll, total_ul_pkts_ll,
total_dl_rate, total_dl_rate_ll, total_dl_pkts_ll, ap_row, ap_stats_col_titles):
def write_port_csv(self, sta_count, ul, dl, ul_pdu, dl_pdu, atten, eid_name, port_data, latency, jitter, tput,
ap_row, ap_stats_col_titles):
row = [self.epoch_time, self.time_stamp(), sta_count,
ul, ul, dl, dl, dl_pdu, dl_pdu, ul_pdu, ul_pdu,
atten, eid_name
]
row = row + [port_data['bps rx'], port_data['bps tx'], port_data['rx-rate'], port_data['tx-rate'],
port_data['signal'], port_data['ap'], port_data['mode'], latency, jitter, total_ul_rate, total_ul_rate_ll, total_ul_pkts_ll, total_dl_rate, total_dl_rate_ll, total_dl_pkts_ll]
port_data['signal'], port_data['ap'], port_data['mode'], latency, jitter, tput]
#Add in info queried from AP. NOTE: do not need to pass in the ap_stats_col_titles
#print("ap_row length {} col_titles length {}".format(len(ap_row),len(self.ap_stats_col_titles)))
@@ -1001,7 +949,7 @@ class L3VariableTime(Realm):
# Submit data to the influx db if configured to do so.
def record_kpi(self, sta_count, ul, dl, ul_pdu, dl_pdu, atten, total_dl_bps, total_ul_bps, total_dl_ll_bps, total_ul_ll_bps):
def record_kpi(self, sta_count, ul, dl, ul_pdu, dl_pdu, atten, total_dl_bps, total_ul_bps):
tags = dict()
tags['requested-ul-bps'] = ul
@@ -1029,8 +977,7 @@ class L3VariableTime(Realm):
row = [self.epoch_time, self.time_stamp(), sta_count,
ul, ul, dl, dl, dl_pdu, dl_pdu, ul_pdu, ul_pdu,
atten,
total_dl_bps, total_ul_bps, (total_ul_bps + total_dl_bps),
total_dl_ll_bps, total_ul_ll_bps, (total_ul_ll_bps + total_dl_ll_bps)
total_dl_bps, total_ul_bps, (total_ul_bps + total_dl_bps)
]
# Add values for any user specified tags
for k in self.user_tags:
@@ -1067,7 +1014,7 @@ class L3VariableTime(Realm):
'UL-Min-Requested','UL-Max-Requested','DL-Min-Requested','DL-Max-Requested',
'UL-Min-PDU','UL-Max-PDU','DL-Min-PDU','DL-Max-PDU','Attenuation',
'Name', 'Rx-Bps', 'Tx-Bps', 'Rx-Link-Rate', 'Tx-Link-Rate', 'RSSI', 'AP', 'Mode',
'Rx-Latency', 'Rx-Jitter', 'Ul-Rx-Goodput-bps', 'Ul-Rx-Rate-ll', 'Ul-Rx-Pkts-ll', 'Dl-Rx-Goodput-bps', 'Dl-Rx-Rate-ll', 'Dl-Rx-Pkts-ll'
'Rx-Latency', 'Rx-Jitter', 'Rx-Goodput-Bps'
]
# Add in columns we are going to query from the AP
for col in self.ap_stats_col_titles:
@@ -1079,8 +1026,7 @@ class L3VariableTime(Realm):
csv_rx_headers = ['Time epoch', 'Time', 'Station-Count',
'UL-Min-Requested','UL-Max-Requested','DL-Min-Requested','DL-Max-Requested',
'UL-Min-PDU','UL-Max-PDU','DL-Min-PDU','DL-Max-PDU','Attenuation',
'Total-Download-Bps', 'Total-Upload-Bps', 'Total-UL/DL-Bps',
'Total-Download-LL-Bps', 'Total-Upload-LL-Bps','Total-UL/DL-LL-Bps'
'Total-Download-Bps', 'Total-Upload-Bps', 'Total-UL/DL-Bps'
]
for k in self.user_tags:
csv_rx_headers.append(k[0])
@@ -1227,7 +1173,7 @@ python3 .\\test_l3_longevity.py --test_duration 4m --endp_type \"lf_tcp lf_udp m
parser.add_argument('--amount_ports_to_reset', help='--amount_ports_to_reset \"<min amount ports> <max amount ports>\" ', default=None)
parser.add_argument('--port_reset_seconds', help='--ports_reset_seconds \"<min seconds> <max seconds>\" ', default="10 30")
parser.add_argument('--lfmgr', help='--lfmgr <hostname for where LANforge GUI is running>',default='localhost')
parser.add_argument('--mgr', help='--mgr <hostname for where LANforge GUI is running>',default='localhost')
parser.add_argument('--test_duration', help='--test_duration <how long to run> example --time 5d (5 days) default: 3m options: number followed by d, h, m or s',default='3m')
parser.add_argument('--tos', help='--tos: Support different ToS settings: BK | BE | VI | VO | numeric',default="BE")
parser.add_argument('--debug', help='--debug flag present debug on enable debugging',action='store_true')
@@ -1243,9 +1189,7 @@ python3 .\\test_l3_longevity.py --test_duration 4m --endp_type \"lf_tcp lf_udp m
parser.add_argument('--ap_read', help='--ap_read flag present enable reading ap', action='store_true')
parser.add_argument('--ap_port', help='--ap_port \'/dev/ttyUSB0\'',default='/dev/ttyUSB0')
parser.add_argument('--ap_baud', help='--ap_baud \'115200\'',default='115200')
# note wl1 is the 5G interface , check the MAC ifconfig -a of the interface to the AP BSSID connection (default may be eth7)
parser.add_argument('--ap_cmd_5g', help='ap_cmd_5g \'wl -i wl1 bs_data\'', default="wl -i wl1 bs_data")
# note wl1 is the 2.4G interface , check the MAC ifconfig -a of the interface to the AP BSSID connection
parser.add_argument('--ap_cmd_2g', help='ap_cmd_2g \'wl -i wl0 bs_data\'', default="wl -i wl0 bs_data")
parser.add_argument('--ap_chanim_cmd_5g', help='ap_chanim_cmd_5g \'wl -i wl1 chanim_stats\'', default="wl -i wl1 chanim_stats")
parser.add_argument('--ap_chanim_cmd_2g', help='ap_chanim_cmd_2g \'w1 -i wl0 chanim_stats\'', default="wl -i wl0 chanim_stats")
@@ -1333,8 +1277,8 @@ python3 .\\test_l3_longevity.py --test_duration 4m --endp_type \"lf_tcp lf_udp m
if args.endp_type:
endp_types = args.endp_type
if args.lfmgr:
lfjson_host = args.lfmgr
if args.mgr:
lfjson_host = args.mgr
if args.upstream_port:
side_b = args.upstream_port
@@ -1366,7 +1310,7 @@ python3 .\\test_l3_longevity.py --test_duration 4m --endp_type \"lf_tcp lf_udp m
influxdb = None
if args.influx_bucket is not None:
from InfluxRequest import RecordInflux
from influx2 import RecordInflux
influxdb = RecordInflux(_influx_host=args.influx_host,
_influx_port=args.influx_port,
_influx_org=args.influx_org,