From 6e7f4e1fb1df43bb3ddb33cc8afdaa98ba390f0c Mon Sep 17 00:00:00 2001 From: Matthew Stidham Date: Thu, 22 Apr 2021 16:54:16 -0700 Subject: [PATCH] Create csv_to_grafana.py and build dependencies Signed-off-by: Matthew Stidham --- py-dashboard/GrafanaRequest.py | 30 +++- py-scripts/csv_to_grafana.py | 264 +++++++++++++++++++++++++++++++++ py-scripts/csv_to_influx.py | 17 ++- py-scripts/grafana_profile.py | 86 ++++++++++- py-scripts/recordinflux.py | 11 +- 5 files changed, 396 insertions(+), 12 deletions(-) create mode 100644 py-scripts/csv_to_grafana.py diff --git a/py-dashboard/GrafanaRequest.py b/py-dashboard/GrafanaRequest.py index fa000224..f51cfbf1 100644 --- a/py-dashboard/GrafanaRequest.py +++ b/py-dashboard/GrafanaRequest.py @@ -53,8 +53,8 @@ class GrafanaRequest: dashboard['title'] = dashboard_name dashboard['tags'] = ['templated'] dashboard['timezone'] = 'browser' - dashboard['shemaVersion'] = 6 - dashboard['version'] = 0 + dashboard['schemaVersion'] = 27 + dashboard['version'] = 4 datastore['dashboard'] = dashboard datastore['overwrite'] = False data = json.dumps(datastore, indent=4) @@ -67,4 +67,28 @@ class GrafanaRequest: def create_dashboard_from_data(self, json_file=None): - pass + self.grafanajson_url = self.grafanajson_url + '/api/dashboards/db' + datastore = dict() + dashboard = dict(json.loads(open(json_file).read())) + datastore['dashboard'] = dashboard + datastore['overwrite'] = False + data = json.dumps(datastore, indent=4) + #return print(data) + return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) + + def create_dashboard_from_dict(self, + dictionary=None): + self.grafanajson_url = self.grafanajson_url + '/api/dashboards/db' + datastore = dict() + dashboard = dict(json.loads(dictionary)) + datastore['dashboard'] = dashboard + datastore['overwrite'] = False + data = json.dumps(datastore, indent=4) + #return print(data) + return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) + + + def create_custom_dashboard(self, + datastore=None): + data = json.dumps(datastore, indent=4) + return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False) \ No newline at end of file diff --git a/py-scripts/csv_to_grafana.py b/py-scripts/csv_to_grafana.py new file mode 100644 index 00000000..b5ad3a65 --- /dev/null +++ b/py-scripts/csv_to_grafana.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 + +import sys +import os +import argparse + +if sys.version_info[0] != 3: + print("This script requires Python 3") + exit(1) + +if 'py-json' not in sys.path: + sys.path.append(os.path.join(os.path.abspath('..'), 'py-json')) + sys.path.append(os.path.join(os.path.abspath('..'), 'py-dashboard')) + +from LANforge.lfcli_base import LFCliBase +import json +from influx2 import RecordInflux +from csv_to_influx import CSVtoInflux, influx_add_parser_args +from grafana_profile import UseGrafana +import random +import string + +class data_to_grafana(LFCliBase): + def __init__(self, + _bucket=None, + _script=None, + _panel_name=None): + self.bucket = _bucket + self.script = _script + self.panel_name = _panel_name + pass + + @property + def json_parser(self): + options = string.ascii_lowercase+string.ascii_uppercase+string.digits + uid = ''.join(random.choice(options) for i in range(9)) + print(uid) + json_dict = { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": True, + "hide": True, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": True, + "gnetId": None, + "graphTooltip": 0, + "id": 10, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": False, + "dashLength": 10, + "dashes": False, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": False, + "id": 3, + "legend": { + "avg": False, + "current": False, + "max": False, + "min": False, + "show": True, + "total": False, + "values": False + }, + "lines": True, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": True + }, + "percentage": False, + "pointradius": 2, + "points": False, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": False, + "steppedLine": False, + "targets": [ + { + "delimiter": ",", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "header": True, + "ignoreUnknown": False, + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: \"%s\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"script\"] == \"%s\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: False)\n |> yield(name: \"mean\")\n " % (self.influx_bucket, self.script), + "refId": "A", + "resultFormat": "time_series", + "schema": [], + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "skipRows": 0, + "tags": [] + } + ], + "thresholds": [], + "timeRegions": [], + "title": self.panel_name, + "tooltip": { + "shared": True, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": None, + "mode": "time", + "name": None, + "show": True, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": None, + "logBase": 1, + "max": None, + "min": None, + "show": True + }, + { + "format": "short", + "label": None, + "logBase": 1, + "max": None, + "min": None, + "show": True + } + ], + "yaxis": { + "align": False, + "alignLevel": None + } + } + ], + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": str(self.script), + "uid": uid + } + return json.dumps(json_dict) + + +def main(): + parser = LFCliBase.create_basic_argparse( + prog='csv_to_grafana.py', + formatter_class=argparse.RawTextHelpFormatter, + epilog='''Auto-create Grafana dashboard from a CSV''', + description='''\ + csv_to_grafana.py + -------------------- + Command example: + ./csv_to_grafana.py + --grafana_token + --influx_host + --influx_org + --influx_token + --influx_bucket + --target_csv + --panel_name''' + ) + required = parser.add_argument_group('required arguments') + required.add_argument('--grafana_token', help='token to access your Grafana database', required=True) + + optional = parser.add_argument_group('optional arguments') + optional.add_argument('--grafana_port', help='Grafana port if different from 3000', default=3000) + optional.add_argument('--grafana_host', help='Grafana host', default='localhost') + optional.add_argument('--panel_name', help='Custom name of the panel', default=None) + + influx_add_parser_args(parser) + + # This argument is specific to this script, so not part of the generic influxdb parser args + # method above. + parser.add_argument('--target_csv', help='CSV file to record to influx database', default="") + + args = parser.parse_args() + + influxdb = RecordInflux(_influx_host=args.influx_host, + _influx_port=args.influx_port, + _influx_org=args.influx_org, + _influx_token=args.influx_token, + _influx_bucket=args.influx_bucket) + + csvtoinflux = CSVtoInflux(influxdb=influxdb, + target_csv=args.target_csv, + _influx_tag=args.influx_tag) + + GrafanaDB = UseGrafana(args.grafana_token, + args.grafana_port, + args.grafana_host) + + scriptname = csvtoinflux.script_name() + + if args.panel_name is None: + panel_name = scriptname + else: + panel_name = args.panel_name + + DataToGrafana = data_to_grafana(_bucket=args.influx_bucket, + _script=scriptname, + _panel_name=panel_name) + + csvtoinflux.post_to_influx() + + grafana_input = DataToGrafana.json_parser + + GrafanaDB.GR.create_dashboard_from_dict(dictionary=grafana_input) diff --git a/py-scripts/csv_to_influx.py b/py-scripts/csv_to_influx.py index 28cfafb9..6135dc5c 100755 --- a/py-scripts/csv_to_influx.py +++ b/py-scripts/csv_to_influx.py @@ -75,9 +75,7 @@ class CSVtoInflux(Realm): date = datetime.datetime.utcfromtimestamp(int(date) / 1000).isoformat() #convert to datetime so influx can read it, this is required numeric_score = line[numeric_score_index] numeric_score = float(numeric_score) #convert to float, InfluxDB cannot - test_details = line[test_details_index] short_description = line[short_description_index] - test_id = line[test_id_index] tags = dict() tags['script'] = line[test_id_index] tags['short-description'] = line[short_description_index] @@ -92,6 +90,21 @@ class CSVtoInflux(Realm): # variable n ame, value, tags, date # total-download-mbps-speed-for-the-duration-of-this-iteration 171.085494 {'script': 'WiFi Capacity'} 2021-04-14T19:04:04.902000 + def script_name(self): + with open(self.target_csv) as fp: + line = fp.readline() + line = line.split('\t') + test_id_index = line.index('test-id') + line = fp.readline() + return line[test_id_index] + + + def create_dashboard(self, + dashboard_name=None): + #Create a dashboard in Grafana to look at the data you just posted to Influx + dashboard_name + + def main(): lfjson_host = "localhost" diff --git a/py-scripts/grafana_profile.py b/py-scripts/grafana_profile.py index 423c55a6..ecdfa09f 100755 --- a/py-scripts/grafana_profile.py +++ b/py-scripts/grafana_profile.py @@ -14,7 +14,7 @@ if 'py-json' not in sys.path: from GrafanaRequest import GrafanaRequest from LANforge.lfcli_base import LFCliBase - +import json class UseGrafana(LFCliBase): def __init__(self, @@ -42,6 +42,83 @@ class UseGrafana(LFCliBase): def list_dashboards(self): return self.GR.list_dashboards() + def create_dashboard_from_data(self, + json_file): + return self.GR.create_dashboard_from_data(json_file=json_file) + + def create_custom_dashboard(self): + name = input('Name of your dashboard: ') + number = input('Number of panels on dashboard: ') + dashboards = dict() + dashboards['title'] = name + dashboards['panels'] = list() + input1 = dict() + input1['name'] = "DS_INFLUXDB" + input1['label'] = "InfluxDB" + input1['description'] = "" + input1['type'] = "datasource" + input1['pluginId'] = "influxdb" + input1['pluginName'] = "InfluxDB" + inputs = list() + inputs.append(input1) + dashboards['__inputs'] = inputs + + for dashboard in range(0,int(number)): + title = input('Title of dashboard # %s ' % dashboard) + lines = input('Lines on your graph? Y or N: ') + target_number = input('How many targets on your graph? ') + if lines == 'Y': + lines = True + linewidth = 1 + else: + lines = False + linewidth = 0 + panel = dict() + panel['aliasColors'] = dict() + panel['bars'] = False + panel['dashLength'] = 10 + fieldConfig = dict() + fieldConfig['defaults'] = dict() + fieldConfig['overrides'] = list() + panel['fieldConfig'] = fieldConfig + panel['fill'] = 1 + panel['fieldGradient'] = 0 + panel['hiddenSeries'] = False + panel['id'] = dashboard + panel['lines'] = lines + panel['linewidth'] = linewidth + panel['nullPointMode'] = None + panel['percentage'] = False + panel['pluginVersion'] = "7.5.4" + panel['stack'] = False + targets = list() + for target in range(0,int(target_number)): + measurement = input('Name of your first measurement') + target_item = dict() + target_item['delimiter'] = ',' + target_item['header'] = True + target_item['ignoreUnknown'] = False + target_item['measurement'] = measurement + target_item['orderByTime'] = "ASC" + target_item['policy'] = "default" + target_item['skipRows'] = 0 + targets.append(target_item) + panel['targets'] = targets + panel['title'] = title + xaxis=dict() + xaxis['buckets'] = None + xaxis['mode'] = "time" + xaxis['name'] = None + xaxis['show'] = True + xaxis['values'] = list() + panel['xaxis'] = xaxis + yaxis = dict() + yaxis['align'] = False + yaxis['alignLevel'] = None + panel['yaxis'] = yaxis + dashboards['panels'].append(panel) + print(json.dumps(dashboards, indent=2)) + return self.GR.create_dashboard_from_dict(dictionary=json.dumps(dashboards)) def main(): parser = LFCliBase.create_basic_argparse( @@ -67,6 +144,8 @@ def main(): optional.add_argument('--grafana_port', help='Grafana port if different from 3000', default=3000) optional.add_argument('--grafana_host', help='Grafana host', default='localhost') optional.add_argument('--list_dashboards', help='List dashboards on Grafana server', default=None) + optional.add_argument('--dashboard_json', help='JSON of existing Grafana dashboard to import', default=None) + optional.add_argument('--create_custom', help='Guided Dashboard creation', default=None) args = parser.parse_args() Grafana = UseGrafana(args.grafana_token, @@ -82,6 +161,11 @@ def main(): if args.list_dashboards is not None: Grafana.list_dashboards() + if args.dashboard_json is not None: + Grafana.create_dashboard_from_data(args.dashboard_json) + + if args.create_custom is not None: + Grafana.create_custom_dashboard() if __name__ == "__main__": main() diff --git a/py-scripts/recordinflux.py b/py-scripts/recordinflux.py index 70b7e337..2e4921e4 100755 --- a/py-scripts/recordinflux.py +++ b/py-scripts/recordinflux.py @@ -55,7 +55,7 @@ def main(): args = parser.parse_args() monitor_interval = LFCliBase.parse_time(args.monitor_interval).total_seconds() longevity = LFCliBase.parse_time(args.longevity).total_seconds() - tags=dict() + tags = dict() tags['script'] = 'recordinflux' if args.influx_user is None: from influx2 import RecordInflux @@ -72,14 +72,13 @@ def main(): else: from influx import RecordInflux grapher = RecordInflux(_influx_host=args.mgr, - _port=args.mgr_port, + _influx_port=args.mgr_port, _influx_db=args.influx_db, _influx_user=args.influx_user, _influx_passwd=args.influx_passwd) - grapher.getdata(longevity=longevity, - devices=args.device, - monitor_interval=monitor_interval, - target_kpi=args.target_kpi) + grapher.monitor_port_data(longevity=longevity, + devices=args.device, + monitor_interval=monitor_interval) if __name__ == "__main__":