wlan-lanforge-scripts/py-scripts/csv_to_influx.py

#!/usr/bin/env python3

#Copies the data from a CSV file from the KPI file generated from a Wifi Capacity test to an Influx database

# The CSV requires three columns in order to work: Date, test details, and numeric-score.

# Date is a unix timestamp, test details is the variable each datapoint is measuring, and numeric-score is the value for that timepoint and variable.

import sys
import os
from pprint import pprint

if sys.version_info[0] != 3:
    print("This script requires Python 3")
    exit(1)

if 'py-json' not in sys.path:
    sys.path.append(os.path.join(os.path.abspath('..'), 'py-json'))

import argparse
from realm import Realm
import pandas as pd


class CSVtoInflux(Realm):
    def __init__(self,
                 lfclient_host="localhost",
                 lfclient_port=8080,
                 debug=False,
                 _exit_on_error=False,
                 _exit_on_fail=False,
                 _proxy_str=None,
                 _capture_signal_list=[],
                 influxdb=None,
                 #_influx_tag=[],
                 target_csv=None):
        super().__init__(lfclient_host=lfclient_host,
                         lfclient_port=lfclient_port,
                         debug_=debug,
                         _exit_on_error=_exit_on_error,
                         _exit_on_fail=_exit_on_fail,
                         _proxy_str=_proxy_str,
                         _capture_signal_list=_capture_signal_list)
        self.influxdb = influxdb
        self.target_csv = target_csv
        #self.influx_tag = _influx_tag

    # Submit data to the influx db if configured to do so.
    def post_to_influx(self):
        df = pd.read_csv(self.target_csv, sep='\t')
        df['Date'] = pd.to_datetime(df['Date'], unit='s')
        df['Date'] = [str(timestamp.isoformat()) for timestamp in df['Date']]
        dates = list(set(df['Date']))
        scriptname=df['test-id'][0]
        for date in dates:
            kpi2 = df[df['Date'] == date][['Date', 'test details', 'numeric-score', 'test-id']]
            metrics = list(set(kpi2['test details']))
            targets = dict()
            for k in metrics:
                targets[k] = [*kpi2[kpi2['test details'] == k]['numeric-score']][0]
                targets[k.replace(' ', '-').lower()] = targets.pop(k)
            targets
            tags = dict()
            tags['script'] = scriptname
            #for item in self.influx_tag:
                #tags[item[0]] = item[1]
            for k in targets.keys():
                self.influxdb.post_to_influx(k, targets[k], tags, date)


def main():
    lfjson_host = "localhost"
    lfjson_port = 8080
    endp_types = "lf_udp"
    debug = False

    parser = argparse.ArgumentParser(
        prog='test_l3_longevity.py',
        # formatter_class=argparse.RawDescriptionHelpFormatter,
        formatter_class=argparse.RawTextHelpFormatter,
        epilog='''
            ''',

        description='''\
csv_to_influx.py:
--------------------

Summary : 
----------
Copies the data from a CSV file generated by a wifi capacity test to an influx database.

Column names are designed for the KPI file  generated by our Wifi Capacity Test.

A user can of course change the column names to match these in order to input any csv file.

The CSV file needs to have the following columns:
    --date - which is a UNIX timestamp
    --test details - which is the variable being measured by the test
    --numeric-score - which is the value for that variable at that point in time.

Generic command layout:
-----------------------
python .\\csv_to_influx.py


Command:
python3 csv_to_influx.py --influx_host localhost --influx_org Candela --influx_token random_token --influx_bucket lanforge
    --target_csv kpi.csv


        ''')

    parser.add_argument('--influx_host', help='Hostname for the Influx database')
    parser.add_argument('--influx_port', help='IP Port for the Influx database', default=8086)
    parser.add_argument('--influx_org', help='Organization for the Influx database')
    parser.add_argument('--influx_token', help='Token for the Influx database')
    parser.add_argument('--influx_bucket', help='Name of the Influx bucket')
    parser.add_argument('--target_csv', help='CSV file to record to influx database', required=True)
    parser.add_argument('--influx_tag', action='append', nargs=2, help='--influx_tag <key> <val>   Can add more than one of these.')

    args = parser.parse_args()

    influxdb = None
    if args.influx_bucket is not None:
        from influx2 import RecordInflux
        influxdb = RecordInflux(_lfjson_host=lfjson_host,
                                _lfjson_port=lfjson_port,
                                _influx_host=args.influx_host,
                                _influx_port=args.influx_port,
                                _influx_org=args.influx_org,
                                _influx_token=args.influx_token,
                                _influx_bucket=args.influx_bucket)
                                #_influx_tag=args.influx_tag)

    csvtoinflux = CSVtoInflux(influxdb=influxdb,
                              target_csv=args.target_csv)
    csvtoinflux.post_to_influx()


if __name__ == "__main__":
    main()