mirror of
https://github.com/Telecominfraproject/wlan-lanforge-scripts.git
synced 2025-10-31 18:58:01 +00:00
Create csv_to_grafana.py and build dependencies
Signed-off-by: Matthew Stidham <stidmatt@protonmail.com>
This commit is contained in:
Matthew Stidham
@@ -53,8 +53,8 @@ class GrafanaRequest:
|
||||
dashboard['title'] = dashboard_name
|
||||
dashboard['tags'] = ['templated']
|
||||
dashboard['timezone'] = 'browser'
|
||||
dashboard['shemaVersion'] = 6
|
||||
dashboard['version'] = 0
|
||||
dashboard['schemaVersion'] = 27
|
||||
dashboard['version'] = 4
|
||||
datastore['dashboard'] = dashboard
|
||||
datastore['overwrite'] = False
|
||||
data = json.dumps(datastore, indent=4)
|
||||
@@ -67,4 +67,28 @@ class GrafanaRequest:
|
||||
|
||||
def create_dashboard_from_data(self,
|
||||
json_file=None):
|
||||
pass
|
||||
self.grafanajson_url = self.grafanajson_url + '/api/dashboards/db'
|
||||
datastore = dict()
|
||||
dashboard = dict(json.loads(open(json_file).read()))
|
||||
datastore['dashboard'] = dashboard
|
||||
datastore['overwrite'] = False
|
||||
data = json.dumps(datastore, indent=4)
|
||||
#return print(data)
|
||||
return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False)
|
||||
|
||||
def create_dashboard_from_dict(self,
|
||||
dictionary=None):
|
||||
self.grafanajson_url = self.grafanajson_url + '/api/dashboards/db'
|
||||
datastore = dict()
|
||||
dashboard = dict(json.loads(dictionary))
|
||||
datastore['dashboard'] = dashboard
|
||||
datastore['overwrite'] = False
|
||||
data = json.dumps(datastore, indent=4)
|
||||
#return print(data)
|
||||
return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False)
|
||||
|
||||
|
||||
def create_custom_dashboard(self,
|
||||
datastore=None):
|
||||
data = json.dumps(datastore, indent=4)
|
||||
return requests.post(self.grafanajson_url, headers=self.headers, data=data, verify=False)
|
||||
264
py-scripts/csv_to_grafana.py
Normal file
264
py-scripts/csv_to_grafana.py
Normal file
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
|
||||
if sys.version_info[0] != 3:
|
||||
print("This script requires Python 3")
|
||||
exit(1)
|
||||
|
||||
if 'py-json' not in sys.path:
|
||||
sys.path.append(os.path.join(os.path.abspath('..'), 'py-json'))
|
||||
sys.path.append(os.path.join(os.path.abspath('..'), 'py-dashboard'))
|
||||
|
||||
from LANforge.lfcli_base import LFCliBase
|
||||
import json
|
||||
from influx2 import RecordInflux
|
||||
from csv_to_influx import CSVtoInflux, influx_add_parser_args
|
||||
from grafana_profile import UseGrafana
|
||||
import random
|
||||
import string
|
||||
|
||||
class data_to_grafana(LFCliBase):
|
||||
def __init__(self,
|
||||
_bucket=None,
|
||||
_script=None,
|
||||
_panel_name=None):
|
||||
self.bucket = _bucket
|
||||
self.script = _script
|
||||
self.panel_name = _panel_name
|
||||
pass
|
||||
|
||||
@property
|
||||
def json_parser(self):
|
||||
options = string.ascii_lowercase+string.ascii_uppercase+string.digits
|
||||
uid = ''.join(random.choice(options) for i in range(9))
|
||||
print(uid)
|
||||
json_dict = {
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": True,
|
||||
"hide": True,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": True,
|
||||
"gnetId": None,
|
||||
"graphTooltip": 0,
|
||||
"id": 10,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": False,
|
||||
"dashLength": 10,
|
||||
"dashes": False,
|
||||
"datasource": "InfluxDB",
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"hiddenSeries": False,
|
||||
"id": 3,
|
||||
"legend": {
|
||||
"avg": False,
|
||||
"current": False,
|
||||
"max": False,
|
||||
"min": False,
|
||||
"show": True,
|
||||
"total": False,
|
||||
"values": False
|
||||
},
|
||||
"lines": True,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": True
|
||||
},
|
||||
"percentage": False,
|
||||
"pointradius": 2,
|
||||
"points": False,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": False,
|
||||
"steppedLine": False,
|
||||
"targets": [
|
||||
{
|
||||
"delimiter": ",",
|
||||
"groupBy": [
|
||||
{
|
||||
"params": [
|
||||
"$__interval"
|
||||
],
|
||||
"type": "time"
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
"null"
|
||||
],
|
||||
"type": "fill"
|
||||
}
|
||||
],
|
||||
"header": True,
|
||||
"ignoreUnknown": False,
|
||||
"orderByTime": "ASC",
|
||||
"policy": "default",
|
||||
"query": "from(bucket: \"%s\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"script\"] == \"%s\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: False)\n |> yield(name: \"mean\")\n " % (self.influx_bucket, self.script),
|
||||
"refId": "A",
|
||||
"resultFormat": "time_series",
|
||||
"schema": [],
|
||||
"select": [
|
||||
[
|
||||
{
|
||||
"params": [
|
||||
"value"
|
||||
],
|
||||
"type": "field"
|
||||
},
|
||||
{
|
||||
"params": [],
|
||||
"type": "mean"
|
||||
}
|
||||
]
|
||||
],
|
||||
"skipRows": 0,
|
||||
"tags": []
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeRegions": [],
|
||||
"title": self.panel_name,
|
||||
"tooltip": {
|
||||
"shared": True,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": None,
|
||||
"mode": "time",
|
||||
"name": None,
|
||||
"show": True,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": None,
|
||||
"logBase": 1,
|
||||
"max": None,
|
||||
"min": None,
|
||||
"show": True
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": None,
|
||||
"logBase": 1,
|
||||
"max": None,
|
||||
"min": None,
|
||||
"show": True
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": False,
|
||||
"alignLevel": None
|
||||
}
|
||||
}
|
||||
],
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": str(self.script),
|
||||
"uid": uid
|
||||
}
|
||||
return json.dumps(json_dict)
|
||||
|
||||
|
||||
def main():
|
||||
parser = LFCliBase.create_basic_argparse(
|
||||
prog='csv_to_grafana.py',
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
epilog='''Auto-create Grafana dashboard from a CSV''',
|
||||
description='''\
|
||||
csv_to_grafana.py
|
||||
--------------------
|
||||
Command example:
|
||||
./csv_to_grafana.py
|
||||
--grafana_token
|
||||
--influx_host
|
||||
--influx_org
|
||||
--influx_token
|
||||
--influx_bucket
|
||||
--target_csv
|
||||
--panel_name'''
|
||||
)
|
||||
required = parser.add_argument_group('required arguments')
|
||||
required.add_argument('--grafana_token', help='token to access your Grafana database', required=True)
|
||||
|
||||
optional = parser.add_argument_group('optional arguments')
|
||||
optional.add_argument('--grafana_port', help='Grafana port if different from 3000', default=3000)
|
||||
optional.add_argument('--grafana_host', help='Grafana host', default='localhost')
|
||||
optional.add_argument('--panel_name', help='Custom name of the panel', default=None)
|
||||
|
||||
influx_add_parser_args(parser)
|
||||
|
||||
# This argument is specific to this script, so not part of the generic influxdb parser args
|
||||
# method above.
|
||||
parser.add_argument('--target_csv', help='CSV file to record to influx database', default="")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
influxdb = RecordInflux(_influx_host=args.influx_host,
|
||||
_influx_port=args.influx_port,
|
||||
_influx_org=args.influx_org,
|
||||
_influx_token=args.influx_token,
|
||||
_influx_bucket=args.influx_bucket)
|
||||
|
||||
csvtoinflux = CSVtoInflux(influxdb=influxdb,
|
||||
target_csv=args.target_csv,
|
||||
_influx_tag=args.influx_tag)
|
||||
|
||||
GrafanaDB = UseGrafana(args.grafana_token,
|
||||
args.grafana_port,
|
||||
args.grafana_host)
|
||||
|
||||
scriptname = csvtoinflux.script_name()
|
||||
|
||||
if args.panel_name is None:
|
||||
panel_name = scriptname
|
||||
else:
|
||||
panel_name = args.panel_name
|
||||
|
||||
DataToGrafana = data_to_grafana(_bucket=args.influx_bucket,
|
||||
_script=scriptname,
|
||||
_panel_name=panel_name)
|
||||
|
||||
csvtoinflux.post_to_influx()
|
||||
|
||||
grafana_input = DataToGrafana.json_parser
|
||||
|
||||
GrafanaDB.GR.create_dashboard_from_dict(dictionary=grafana_input)
|
||||
@@ -75,9 +75,7 @@ class CSVtoInflux(Realm):
|
||||
date = datetime.datetime.utcfromtimestamp(int(date) / 1000).isoformat() #convert to datetime so influx can read it, this is required
|
||||
numeric_score = line[numeric_score_index]
|
||||
numeric_score = float(numeric_score) #convert to float, InfluxDB cannot
|
||||
test_details = line[test_details_index]
|
||||
short_description = line[short_description_index]
|
||||
test_id = line[test_id_index]
|
||||
tags = dict()
|
||||
tags['script'] = line[test_id_index]
|
||||
tags['short-description'] = line[short_description_index]
|
||||
@@ -92,6 +90,21 @@ class CSVtoInflux(Realm):
|
||||
# variable n ame, value, tags, date
|
||||
# total-download-mbps-speed-for-the-duration-of-this-iteration 171.085494 {'script': 'WiFi Capacity'} 2021-04-14T19:04:04.902000
|
||||
|
||||
def script_name(self):
|
||||
with open(self.target_csv) as fp:
|
||||
line = fp.readline()
|
||||
line = line.split('\t')
|
||||
test_id_index = line.index('test-id')
|
||||
line = fp.readline()
|
||||
return line[test_id_index]
|
||||
|
||||
|
||||
def create_dashboard(self,
|
||||
dashboard_name=None):
|
||||
#Create a dashboard in Grafana to look at the data you just posted to Influx
|
||||
dashboard_name
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
lfjson_host = "localhost"
|
||||
|
||||
@@ -14,7 +14,7 @@ if 'py-json' not in sys.path:
|
||||
|
||||
from GrafanaRequest import GrafanaRequest
|
||||
from LANforge.lfcli_base import LFCliBase
|
||||
|
||||
import json
|
||||
|
||||
class UseGrafana(LFCliBase):
|
||||
def __init__(self,
|
||||
@@ -42,6 +42,83 @@ class UseGrafana(LFCliBase):
|
||||
def list_dashboards(self):
|
||||
return self.GR.list_dashboards()
|
||||
|
||||
def create_dashboard_from_data(self,
|
||||
json_file):
|
||||
return self.GR.create_dashboard_from_data(json_file=json_file)
|
||||
|
||||
def create_custom_dashboard(self):
|
||||
name = input('Name of your dashboard: ')
|
||||
number = input('Number of panels on dashboard: ')
|
||||
dashboards = dict()
|
||||
dashboards['title'] = name
|
||||
dashboards['panels'] = list()
|
||||
input1 = dict()
|
||||
input1['name'] = "DS_INFLUXDB"
|
||||
input1['label'] = "InfluxDB"
|
||||
input1['description'] = ""
|
||||
input1['type'] = "datasource"
|
||||
input1['pluginId'] = "influxdb"
|
||||
input1['pluginName'] = "InfluxDB"
|
||||
inputs = list()
|
||||
inputs.append(input1)
|
||||
dashboards['__inputs'] = inputs
|
||||
|
||||
for dashboard in range(0,int(number)):
|
||||
title = input('Title of dashboard # %s ' % dashboard)
|
||||
lines = input('Lines on your graph? Y or N: ')
|
||||
target_number = input('How many targets on your graph? ')
|
||||
if lines == 'Y':
|
||||
lines = True
|
||||
linewidth = 1
|
||||
else:
|
||||
lines = False
|
||||
linewidth = 0
|
||||
panel = dict()
|
||||
panel['aliasColors'] = dict()
|
||||
panel['bars'] = False
|
||||
panel['dashLength'] = 10
|
||||
fieldConfig = dict()
|
||||
fieldConfig['defaults'] = dict()
|
||||
fieldConfig['overrides'] = list()
|
||||
panel['fieldConfig'] = fieldConfig
|
||||
panel['fill'] = 1
|
||||
panel['fieldGradient'] = 0
|
||||
panel['hiddenSeries'] = False
|
||||
panel['id'] = dashboard
|
||||
panel['lines'] = lines
|
||||
panel['linewidth'] = linewidth
|
||||
panel['nullPointMode'] = None
|
||||
panel['percentage'] = False
|
||||
panel['pluginVersion'] = "7.5.4"
|
||||
panel['stack'] = False
|
||||
targets = list()
|
||||
for target in range(0,int(target_number)):
|
||||
measurement = input('Name of your first measurement')
|
||||
target_item = dict()
|
||||
target_item['delimiter'] = ','
|
||||
target_item['header'] = True
|
||||
target_item['ignoreUnknown'] = False
|
||||
target_item['measurement'] = measurement
|
||||
target_item['orderByTime'] = "ASC"
|
||||
target_item['policy'] = "default"
|
||||
target_item['skipRows'] = 0
|
||||
targets.append(target_item)
|
||||
panel['targets'] = targets
|
||||
panel['title'] = title
|
||||
xaxis=dict()
|
||||
xaxis['buckets'] = None
|
||||
xaxis['mode'] = "time"
|
||||
xaxis['name'] = None
|
||||
xaxis['show'] = True
|
||||
xaxis['values'] = list()
|
||||
panel['xaxis'] = xaxis
|
||||
yaxis = dict()
|
||||
yaxis['align'] = False
|
||||
yaxis['alignLevel'] = None
|
||||
panel['yaxis'] = yaxis
|
||||
dashboards['panels'].append(panel)
|
||||
print(json.dumps(dashboards, indent=2))
|
||||
return self.GR.create_dashboard_from_dict(dictionary=json.dumps(dashboards))
|
||||
|
||||
def main():
|
||||
parser = LFCliBase.create_basic_argparse(
|
||||
@@ -67,6 +144,8 @@ def main():
|
||||
optional.add_argument('--grafana_port', help='Grafana port if different from 3000', default=3000)
|
||||
optional.add_argument('--grafana_host', help='Grafana host', default='localhost')
|
||||
optional.add_argument('--list_dashboards', help='List dashboards on Grafana server', default=None)
|
||||
optional.add_argument('--dashboard_json', help='JSON of existing Grafana dashboard to import', default=None)
|
||||
optional.add_argument('--create_custom', help='Guided Dashboard creation', default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
Grafana = UseGrafana(args.grafana_token,
|
||||
@@ -82,6 +161,11 @@ def main():
|
||||
if args.list_dashboards is not None:
|
||||
Grafana.list_dashboards()
|
||||
|
||||
if args.dashboard_json is not None:
|
||||
Grafana.create_dashboard_from_data(args.dashboard_json)
|
||||
|
||||
if args.create_custom is not None:
|
||||
Grafana.create_custom_dashboard()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -55,7 +55,7 @@ def main():
|
||||
args = parser.parse_args()
|
||||
monitor_interval = LFCliBase.parse_time(args.monitor_interval).total_seconds()
|
||||
longevity = LFCliBase.parse_time(args.longevity).total_seconds()
|
||||
tags=dict()
|
||||
tags = dict()
|
||||
tags['script'] = 'recordinflux'
|
||||
if args.influx_user is None:
|
||||
from influx2 import RecordInflux
|
||||
@@ -72,14 +72,13 @@ def main():
|
||||
else:
|
||||
from influx import RecordInflux
|
||||
grapher = RecordInflux(_influx_host=args.mgr,
|
||||
_port=args.mgr_port,
|
||||
_influx_port=args.mgr_port,
|
||||
_influx_db=args.influx_db,
|
||||
_influx_user=args.influx_user,
|
||||
_influx_passwd=args.influx_passwd)
|
||||
grapher.getdata(longevity=longevity,
|
||||
devices=args.device,
|
||||
monitor_interval=monitor_interval,
|
||||
target_kpi=args.target_kpi)
|
||||
grapher.monitor_port_data(longevity=longevity,
|
||||
devices=args.device,
|
||||
monitor_interval=monitor_interval)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user