From 40fde97861009ee050f83fd27cc19bc13a7ae731 Mon Sep 17 00:00:00 2001 From: Matthew Stidham Date: Tue, 31 Aug 2021 15:55:36 -0700 Subject: [PATCH] Major Grafana Dashboard script restructuring Signed-off-by: Matthew Stidham --- py-dashboard/GhostRequest.py | 13 +- py-dashboard/GrafanaRequest.py | 2 +- py-dashboard/InfluxRequest.py | 119 ++++++++++++++++++ py-json/cv_test_manager.py | 18 +-- py-json/vap_profile.py | 14 +-- py-json/vr_profile2.py | 27 ---- py-scripts/create_vap.py | 61 ++++++--- py-scripts/create_vr.py | 8 +- .../{ => scripts_deprecated}/csv_to_influx.py | 2 + .../{ => scripts_deprecated}/influx2.py | 2 + .../test_ipv4_variable_time.py | 5 +- py-scripts/test_l3_longevity.py | 19 ++- 12 files changed, 210 insertions(+), 80 deletions(-) create mode 100644 py-dashboard/InfluxRequest.py rename py-scripts/{ => scripts_deprecated}/csv_to_influx.py (99%) rename py-scripts/{ => scripts_deprecated}/influx2.py (98%) diff --git a/py-dashboard/GhostRequest.py b/py-dashboard/GhostRequest.py index 9252f8e4..3d8b8deb 100644 --- a/py-dashboard/GhostRequest.py +++ b/py-dashboard/GhostRequest.py @@ -12,20 +12,20 @@ if sys.version_info[0] != 3: exit() import requests - import jwt from datetime import datetime import json import subprocess from scp import SCPClient import paramiko -from GrafanaRequest import GrafanaRequest -from influx2 import RecordInflux import time from collections import Counter import shutil import itertools +from GrafanaRequest import GrafanaRequest +from InfluxRequest import RecordInflux + class CSVReader: def read_csv(self, @@ -439,7 +439,12 @@ class GhostRequest: # create Grafana Dashboard target_files = [] for folder in target_folders: - target_files.append(folder.split('/')[-1] + '/kpi.csv') + target_file=folder.split('/')[-1] + '/kpi.csv' + try: + open(target_file) + target_files.append(target_file) + except: + pass if self.debug: print('Target files: %s' % target_files) diff --git a/py-dashboard/GrafanaRequest.py b/py-dashboard/GrafanaRequest.py index 5b577ff9..727818d2 100644 --- a/py-dashboard/GrafanaRequest.py +++ b/py-dashboard/GrafanaRequest.py @@ -435,7 +435,7 @@ class GrafanaRequest: print('create snapshot') grafanajson_url = self.grafanajson_url + '/api/snapshots' data = self.get_dashboard(title) - data['expires'] = 360000 + data['expires'] = False data['external'] = False data['timeout'] = 15 if self.debug: diff --git a/py-dashboard/InfluxRequest.py b/py-dashboard/InfluxRequest.py new file mode 100644 index 00000000..37838159 --- /dev/null +++ b/py-dashboard/InfluxRequest.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +# pip3 install influxdb-client + +# Version 2.0 influx DB Client + +import sys +import os + +import pandas as pd + +if sys.version_info[0] != 3: + print("This script requires Python 3") + exit(1) + + +if 'py-json' not in sys.path: + sys.path.append(os.path.join(os.path.abspath('..'), 'py-json')) + + +import requests +import json +import influxdb_client +from influxdb_client.client.write_api import SYNCHRONOUS +import time + +import datetime + +def influx_add_parser_args(parser): + parser.add_argument('--influx_host', help='Hostname for the Influx database', default=None) + parser.add_argument('--influx_port', help='IP Port for the Influx database', default=8086) + parser.add_argument('--influx_org', help='Organization for the Influx database', default=None) + parser.add_argument('--influx_token', help='Token for the Influx database', default=None) + parser.add_argument('--influx_bucket', help='Name of the Influx bucket', default=None) + parser.add_argument('--influx_tag', action='append', nargs=2, + help='--influx_tag Can add more than one of these.', default=[]) + +class RecordInflux(): + def __init__(self, + _influx_host="localhost", + _influx_port=8086, + _influx_org=None, + _influx_token=None, + _influx_bucket=None, + _debug_on=False, + _exit_on_fail=False): + self.influx_host = _influx_host + self.influx_port = _influx_port + self.influx_org = _influx_org + self.influx_token = _influx_token + self.influx_bucket = _influx_bucket + self.url = "http://%s:%s" % (self.influx_host, self.influx_port) + self.client = influxdb_client.InfluxDBClient(url=self.url, + token=self.influx_token, + org=self.influx_org, + debug=_debug_on) + self.write_api = self.client.write_api(write_options=SYNCHRONOUS) + + def post_to_influx(self, key, value, tags, time): + p = influxdb_client.Point(key) + for tag_key, tag_value in tags.items(): + p.tag(tag_key, tag_value) + print(tag_key, tag_value) + p.time(time) + p.field("value", value) + self.write_api.write(bucket=self.influx_bucket, org=self.influx_org, record=p) + + def csv_to_influx(self, csv): + df = pd.read_csv(csv, sep='\t') + df['Date'] = [datetime.datetime.utcfromtimestamp(int(date) / 1000).isoformat() for date in df['Date']] + items = list(df.reset_index().transpose().to_dict().values()) + influx_variables = ['script', 'short-description', 'test_details', 'Graph-Group', + 'DUT-HW-version', 'DUT-SW-version', 'DUT-Serial-Num', 'testbed', 'Test Tag', 'Units'] + csv_variables = ['test-id', 'short-description', 'test details', 'Graph-Group', + 'dut-hw-version', 'dut-sw-version', 'dut-serial-num', 'test-rig', 'test-tag', 'Units'] + csv_vs_influx = dict(zip(csv_variables, influx_variables)) + columns = list(df.columns) + for item in items: + tags = dict() + short_description = item['short-description'] + numeric_score = item['numeric-score'] + date = item['Date'] + for variable in csv_variables: + if variable in columns: + influx_variable = csv_vs_influx[variable] + tags[influx_variable] = item[variable] + self.post_to_influx(short_description, numeric_score, tags, date) + + + def set_bucket(self, b): + self.influx_bucket = b + + # Don't use this unless you are sure you want to. + # More likely you would want to generate KPI in the + # individual test cases and poke those relatively small bits of + # info into influxdb. + # This will not end until the 'longevity' timer has expired. + # This function pushes data directly into the Influx database and defaults to all columns. + def monitor_port_data(self, + lanforge_host="localhost", + devices=None, + longevity=None, + monitor_interval=None, + bucket=None, + tags=None): # dict + url = 'http://' + lanforge_host + ':8080/port/1/1/' + end = datetime.datetime.now() + datetime.timedelta(0, longevity) + while datetime.datetime.now() < end: + for station in devices: + url1 = url + station + response = json.loads(requests.get(url1).text) + + current_time = str(datetime.datetime.utcnow().isoformat()) + + # Poke everything into influx db + for key in response['interface'].keys(): + self.post_to_influx("%s-%s" % (station, key), response['interface'][key], tags, current_time) + + time.sleep(monitor_interval) diff --git a/py-json/cv_test_manager.py b/py-json/cv_test_manager.py index 54647fa6..fa9a89fe 100644 --- a/py-json/cv_test_manager.py +++ b/py-json/cv_test_manager.py @@ -4,6 +4,11 @@ Note: This script is working as library for chamberview tests. """ import time +import sys +import os + +if 'py-dashboard' not in sys.path: + sys.path.append(os.path.join(os.path.abspath('..'), 'py-dashboard')) from LANforge.lfcli_base import LFCliBase from realm import Realm @@ -11,7 +16,7 @@ import json from pprint import pprint import argparse from cv_test_reports import lanforge_reports as lf_rpt -from csv_to_influx import * +from InfluxRequest import * import os.path @@ -414,18 +419,15 @@ class cv_test(Realm): # lf_wifi_capacity_test.py may be run / initiated by a remote system against a lanforge # the local_lf_report_dir is data is stored, if there is no local_lf_report_dir then the test is run directly on lanforge if self.local_lf_report_dir == "": - path = "%s/kpi.csv" % (self.lf_report_dir) + csv_path = "%s/kpi.csv" % (self.lf_report_dir) else: kpi_location = self.local_lf_report_dir + "/" + os.path.basename(self.lf_report_dir) # the local_lf_report_dir is the parent directory, need to get the directory name - path = "%s/kpi.csv" % (kpi_location) + csv_path = "%s/kpi.csv" % (kpi_location) - print("Attempt to submit kpi: ", path) - csvtoinflux = CSVtoInflux(influxdb=influxdb, - target_csv=path, - _influx_tag=args.influx_tag) + print("Attempt to submit kpi: ", csv_path) print("Posting to influx...\n") - csvtoinflux.post_to_influx() + influxdb.csv_to_influx(csv_path) print("All done posting to influx.\n") diff --git a/py-json/vap_profile.py b/py-json/vap_profile.py index 7ddeb664..573dc157 100644 --- a/py-json/vap_profile.py +++ b/py-json/vap_profile.py @@ -214,13 +214,13 @@ class VAPProfile(LFCliBase): def create(self, resource, radio, channel=None, up_=None, debug=False, use_ht40=True, use_ht80=True, use_ht160=False, country=0, suppress_related_commands_=True, use_radius=False, hs20_enable=False, bridge=True): - port_list = self.local_realm.json_get("port/1/1/list") - if port_list is not None: - port_list = port_list['interfaces'] - for port in port_list: - for k, v in port.items(): - if v['alias'] == self.vap_name: - self.local_realm.rm_port(v['port'], check_exists=True) + #port_list = self.local_realm.json_get("port/1/1/list") + #if port_list is not None: + # port_list = port_list['interfaces'] + # for port in port_list: + # for k, v in port.items(): + # if v['alias'] == self.vap_name: + # self.local_realm.rm_port(v['port'], check_exists=True) if use_ht160: self.desired_add_vap_flags.append("enable_80211d") self.desired_add_vap_flags_mask.append("enable_80211d") diff --git a/py-json/vr_profile2.py b/py-json/vr_profile2.py index 082c874c..0e37756c 100644 --- a/py-json/vr_profile2.py +++ b/py-json/vr_profile2.py @@ -2,8 +2,6 @@ import time from pprint import pprint from random import randint -from geometry import Rect, Group - from LANforge import LFUtils from base_profile import BaseProfile @@ -64,31 +62,6 @@ class VRProfile(BaseProfile): occupied_area = self.get_occupied_area(resource=resource, debug=debug) return Rect(x=0, y=0, height=occupied_area.height, width=occupied_area.width) - def get_all_vrcx_bounds(self, resource=None, debug=False): - """ - Computes bounds of all free vrcx ports but omits Virtual Routers - :param resource: - :param debug: - :return: rectangle encompasing all free vrcx ports or None - """ - if (resource is None) or (resource < 1): - raise ValueError("get_netsmith_bounds wants resource id") - vrcx_map = self.vrcx_list(resource=resource, debug=debug) - rect_list = [] - for eid,item in vrcx_map.items(): - rect_list.append(self.vr_to_rect(item)) - if len(rect_list) < 1: - return None - bounding_group = Group() - for item in rect_list: - bounding_group.append(item) - - bounding_group.update() - - return Rect(x=bounding_group.x, - y=bounding_group.y, - width=bounding_group.width, - height=bounding_group.height) def vr_eid_to_url(self, eid_str=None, debug=False): debug |= self.debug diff --git a/py-scripts/create_vap.py b/py-scripts/create_vap.py index 9b5f73fc..8bddae17 100755 --- a/py-scripts/create_vap.py +++ b/py-scripts/create_vap.py @@ -32,6 +32,7 @@ class CreateVAP(Realm): _host=None, _port=None, _vap_list=None, + _resource=None, _vap_flags=None, _mode=None, _number_template="00000", @@ -53,6 +54,7 @@ class CreateVAP(Realm): self.security = _security self.password = _password self.vap_list = _vap_list + self.resource = _resource if _vap_flags is None: self.vap_flags = ["wpa2_enable", "80211u_enable", "create_admin_down"] else: @@ -85,7 +87,7 @@ class CreateVAP(Realm): self.vap_profile.use_security(self.security, self.ssid, passwd=self.password) print("Creating VAPs") - self.vap_profile.create(resource = 1, + self.vap_profile.create(resource = self.resource, radio = self.radio, channel = self.channel, country=self.country_code, @@ -124,7 +126,7 @@ Command example: ''') optional = parser.add_argument_group('optional arguments') - optional.add_argument('--num_vaps', help='Number of VAPs to Create', required=False) + optional.add_argument('--num_vaps', help='Number of VAPs to Create', required=False, default=1) optional.add_argument('--vap_flag', help='VAP flags to add', required=False, default=None, action='append') optional.add_argument('--bridge', help='Create a bridge connecting the VAP to a port', required=False, default=False) optional.add_argument('--mac', help='Custom mac address', default="xx:xx:xx:xx:*:xx") @@ -132,6 +134,9 @@ Command example: optional.add_argument('--channel', default=36) optional.add_argument('--country_code', default=0) optional.add_argument('--nss', default=False) + optional.add_argument('--resource', default=1) + optional.add_argument('--start_id', default=0) + optional.add_argument('--vap_name',default=None) args = parser.parse_args() #if args.debug: # pprint.pprint(args) @@ -139,35 +144,55 @@ Command example: if (args.radio is None): raise ValueError("--radio required") - num_vap = 1 - if (args.num_vaps is not None) and (int(args.num_vaps) > 0): - num_vaps_converted = int(args.num_vaps) - num_vap = num_vaps_converted + num_vap = int(args.num_vaps) vap_list = LFUtils.port_name_series(prefix="vap", - start_id=0, + start_id=int(args.start_id), end_id=num_vap-1, padding_number=10000, radio=args.radio) - print(args.passwd) - print(args.ssid) + #print(args.passwd) + #print(args.ssid) - for vap in vap_list: + if args.vap_name is None: + for vap in vap_list: + create_vap = CreateVAP(_host=args.mgr, + _port=args.mgr_port, + _ssid=args.ssid, + _password=args.passwd, + _security=args.security, + _mode=args.mode, + _vap_list=vap, + _resource=args.resource, + _vap_flags=args.vap_flag, + _radio=args.radio, + _channel=args.channel, + _country_code=args.country_code, + _nss=args.nss, + _proxy_str=args.proxy, + _bridge=args.bridge, + _debug_on=args.debug) + print('Creating VAP') + + create_vap.build() + else: + vap_name = "vap"+args.vap_name create_vap = CreateVAP(_host=args.mgr, - _port=args.mgr_port, - _ssid=args.ssid, - _password=args.passwd, - _security=args.security, + _port=args.mgr_port, + _ssid=args.ssid, + _password=args.passwd, + _security=args.security, _mode=args.mode, - _vap_list=vap, + _vap_list=vap_name, + _resource=args.resource, _vap_flags=args.vap_flag, - _radio=args.radio, + _radio=args.radio, _channel=args.channel, _country_code=args.country_code, _nss=args.nss, - _proxy_str=args.proxy, + _proxy_str=args.proxy, _bridge=args.bridge, - _debug_on=args.debug) + _debug_on=args.debug) print('Creating VAP') create_vap.build() diff --git a/py-scripts/create_vr.py b/py-scripts/create_vr.py index 5b23685f..dbf27110 100755 --- a/py-scripts/create_vr.py +++ b/py-scripts/create_vr.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ - Script for creating a variable number of bridges. + Script for creating a variable number of virtual routers. """ import os @@ -171,9 +171,9 @@ Command example: create_vr.clean() create_vr.build() create_vr.start() - # create_vr.monitor() - create_vr.stop() - create_vr.clean() + create_vr.monitor() + #create_vr.stop() + #create_vr.clean() print('Created Virtual Router') if __name__ == "__main__": diff --git a/py-scripts/csv_to_influx.py b/py-scripts/scripts_deprecated/csv_to_influx.py similarity index 99% rename from py-scripts/csv_to_influx.py rename to py-scripts/scripts_deprecated/csv_to_influx.py index c69735e4..7d0b9a15 100755 --- a/py-scripts/csv_to_influx.py +++ b/py-scripts/scripts_deprecated/csv_to_influx.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +# DEPRECATED, PLEASE USE InfluxRequest.py INSTEAD + # Copies the data from a CSV file from the KPI file generated from a Wifi Capacity test to an Influx database # The CSV requires three columns in order to work: Date, test details, and numeric-score. diff --git a/py-scripts/influx2.py b/py-scripts/scripts_deprecated/influx2.py similarity index 98% rename from py-scripts/influx2.py rename to py-scripts/scripts_deprecated/influx2.py index ed78f66e..b3e6ad8a 100755 --- a/py-scripts/influx2.py +++ b/py-scripts/scripts_deprecated/influx2.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +# DEPRECATED, PLEASE USE InfluxRequest.py INSTEAD + # pip3 install influxdb-client # Version 2.0 influx DB Client diff --git a/py-scripts/scripts_deprecated/test_ipv4_variable_time.py b/py-scripts/scripts_deprecated/test_ipv4_variable_time.py index 74282ba1..cdf65a85 100755 --- a/py-scripts/scripts_deprecated/test_ipv4_variable_time.py +++ b/py-scripts/scripts_deprecated/test_ipv4_variable_time.py @@ -34,6 +34,9 @@ if sys.version_info[0] != 3: if 'py-json' not in sys.path: sys.path.append(os.path.join(os.path.abspath('../..'), 'py-json')) +if 'py-dashboard' not in sys.path: + sys.path.append(os.path.join(os.path.abspath('../..'), 'py-dashboard')) + import argparse from LANforge import LFUtils from realm import Realm @@ -442,7 +445,7 @@ python3 ./test_ipv4_variable_time.py # manager = args.influx_mgr if args.influx_org is not None: - from influx2 import RecordInflux + from InfluxRequest import RecordInflux grapher = RecordInflux( # _influx_host=manager, _influx_port=args.influx_port, _influx_org=args.influx_org, diff --git a/py-scripts/test_l3_longevity.py b/py-scripts/test_l3_longevity.py index 689e4fb0..0b65d716 100755 --- a/py-scripts/test_l3_longevity.py +++ b/py-scripts/test_l3_longevity.py @@ -47,15 +47,9 @@ INCLUDE_IN_README ''' + import sys import os -from pprint import pprint -from csv_to_influx import * -import re -import serial -import pexpect -from pexpect_serial import SerialSpawn -from lf_report import lf_report if sys.version_info[0] != 3: print("This script requires Python 3") @@ -63,15 +57,20 @@ if sys.version_info[0] != 3: if 'py-json' not in sys.path: sys.path.append(os.path.join(os.path.abspath('..'), 'py-json')) +if 'py-dashboard' not in sys.path: + sys.path.append(os.path.join(os.path.abspath('..'), 'py-dashboard')) +from pprint import pprint +from InfluxRequest import * +import serial +import pexpect +from pexpect_serial import SerialSpawn +from lf_report import lf_report import argparse -#from LANforge.lfcli_base import LFCliBase from LANforge import LFUtils -#import realm from realm import Realm import time import datetime -import subprocess import csv # This class handles running the test and generating reports.