Major Ghost script improvements and additions

Signed-off-by: Matthew Stidham <stidmatt@gmail.com>

py-dashboard/GhostRequest.py

@@ -61,7 +61,7 @@ class CSVReader:
     def to_html(self, df):
         html = ''
         html = html + ('<table style="border:1px solid #ddd"><tr>')
-        for row in df:
+        for row in df[1:]:
             for item in row:
                 html = html + ('<td style="border:1px solid #ddd">%s</td>' % item)
             html = html + ('</tr>\n<tr>')
@@ -75,12 +75,18 @@ class CSVReader:
         for row in df:
             try:
                 if expression == 'less than':
-                    if float(row[target_index]) <= target:
+                    if float(row[target_index]) < target:
                         targets.append(counter)
                         counter += 1
                     else:
                         counter += 1
                 if expression == 'greater than':
+                    if float(row[target_index]) > target:
+                        targets.append(counter)
+                        counter += 1
+                    else:
+                        counter += 1
+                if expression == 'greater than or equal to':
                     if float(row[target_index]) >= target:
                         targets.append(counter)
                         counter += 1
@@ -90,6 +96,12 @@ class CSVReader:
                 counter += 1
         return list(map(df.__getitem__, targets))
 
+    def concat(self, dfs):
+        final_df = dfs[0]
+        for df in dfs[1:]:
+            final_df = final_df + df[1:]
+        return final_df
+
 
 class GhostRequest:
     def __init__(self,
@@ -193,25 +205,26 @@ class GhostRequest:
                          tags='custom',
                          authors=authors)
 
-    def wifi_capacity_to_ghost(self,
-                               authors,
-                               folders,
-                               title=None,
-                               server_pull=None,
-                               ghost_host=None,
-                               port='22',
-                               user_pull='lanforge',
-                               password_pull='lanforge',
-                               user_push=None,
-                               password_push=None,
-                               customer=None,
-                               testbed='Unknown Testbed',
-                               test_run=None,
-                               target_folders=list(),
-                               grafana_dashboard=None,
-                               grafana_token=None,
-                               grafana_host=None,
-                               grafana_port=3000):
+    def kpi_to_ghost(self,
+                     authors,
+                     folders,
+                     parent_folder=None,
+                     title=None,
+                     server_pull=None,
+                     ghost_host=None,
+                     port=22,
+                     user_push=None,
+                     password_push=None,
+                     customer=None,
+                     testbed='Unknown Testbed',
+                     test_run=None,
+                     target_folders=list(),
+                     grafana_dashboard=None,
+                     grafana_token=None,
+                     grafana_host=None,
+                     grafana_port=3000,
+                     grafana_datasource='InfluxDB',
+                     grafana_bucket=None):
         text = ''
         csvreader = CSVReader()
         if grafana_token is not None:
@@ -219,27 +232,44 @@ class GhostRequest:
                                      grafana_host,
                                      grafanajson_port=grafana_port
                                      )
-        if test_run is None:
-            test_run = sorted(folders)[0].split('/')[-1].strip('/')
-        print(folders)
-        for folder in folders:
-            print(folder)
-            ssh_pull = paramiko.SSHClient()
-            ssh_pull.set_missing_host_key_policy(paramiko.client.AutoAddPolicy)
-            ssh_pull.connect(server_pull,
-                             port,
-                             username=user_pull,
-                             password=password_pull,
-                             allow_agent=False,
-                             look_for_keys=False)
-            scp_pull = SCPClient(ssh_pull.get_transport())
-            scp_pull.get(folder, recursive=True)
-            target_folder = str(folder).rstrip('/').split('/')[-1]
-            target_folders.append(target_folder)
-            print('Target folder: %s' % target_folder)
+        #if parent_folder is None:
+            #test_run = sorted(folders)[0].split('/')[-1].strip('/')
+        print('Folders: %s' % folders)
+
+        ssh_push = paramiko.SSHClient()
+        ssh_push.set_missing_host_key_policy(paramiko.client.AutoAddPolicy)
+        ssh_push.connect(ghost_host,
+                         port,
+                         username=user_push,
+                         password=password_push,
+                         allow_agent=False,
+                         look_for_keys=False)
+        scp_push = SCPClient(ssh_push.get_transport())
+
+        if parent_folder is not None:
+            print("parent_folder %s" % parent_folder)
+            files = os.listdir(parent_folder)
+            print(files)
+            for file in files:
+                if os.path.isdir(parent_folder+'/'+file) is True:
+                    import shutil
+                    shutil.move(parent_folder+'/'+file, file)
+                    target_folders.append(file)
+            print('Target folders: %s' % target_folders)
+        else:
+            for folder in folders:
+                print(folder)
+                target_folders.append(folder)
+
+        testbeds = list()
+        pdfs = list()
+        high_priority_list = list()
+        low_priority_list = list()
+        images = list()
+
+        for target_folder in target_folders:
             try:
                 target_file = '%s/kpi.csv' % target_folder
-                print('target file %s' % target_file)
                 df = csvreader.read_csv(file=target_file, sep='\t')
                 csv_testbed = csvreader.get_column(df, 'test-rig')[0]
                 pass_fail = Counter(csvreader.get_column(df, 'pass/fail'))
@@ -249,87 +279,94 @@ class GhostRequest:
                     text = text + 'Percentage of tests passed: %s<br />' % (
                             pass_fail['PASS'] / (pass_fail['PASS'] + pass_fail['FAIL']))
 
-                print(csv_testbed)
             except:
-                pass
+                print("Failure")
+                target_folders.remove(target_folder)
+                break
             if len(csv_testbed) > 2:
                 testbed = csv_testbed
-                text = text + 'Testbed: %s<br />' % testbed
+                testbeds.append(testbed)
             if testbed == 'Unknown Testbed':
                 raise UserWarning('Please define your testbed')
-            print('testbed %s' % testbed)
 
-            ssh_push = paramiko.SSHClient()
-            ssh_push.set_missing_host_key_policy(paramiko.client.AutoAddPolicy)
-            ssh_push.connect(ghost_host,
-                             port,
-                             username=user_push,
-                             password=password_push,
-                             allow_agent=False,
-                             look_for_keys=False)
-            scp_push = SCPClient(ssh_push.get_transport())
             local_path = '/home/%s/%s/%s' % (user_push, customer, testbed)
-            transport = paramiko.Transport((ghost_host, port))
+
+            transport = paramiko.Transport(ghost_host, port)
             transport.connect(None, user_push, password_push)
             sftp = paramiko.sftp_client.SFTPClient.from_transport(transport)
-            print('Local Path: %s' % local_path)
-            try:
-                sftp.mkdir(local_path)
-            except:
-                print('folder %s already exists' % local_path)
-            scp_push.put(target_folder, recursive=True, remote_path=local_path)
-            print(local_path + '/' + target_folder)
+
+            print(local_path)
+            print(target_folder)
+            scp_push.put(target_folder, local_path, recursive=True)
             files = sftp.listdir(local_path + '/' + target_folder)
-            # print('Files: %s' % files)
             for file in files:
                 if 'pdf' in file:
-                    url = 'http://%s/%s/%s/%s/%s/%s' % (
-                        ghost_host, customer.strip('/'), testbed, test_run, target_folder, file)
-                    text = text + 'PDF of results: <a href="%s">%s</a><br />' % (url, file)
-                    print(url)
-            scp_pull.close()
+                    url = 'http://%s/%s/%s/%s/%s' % (
+                        ghost_host, customer.strip('/'), testbed, test_run, file)
+                    pdfs.append('PDF of results: <a href="%s">%s</a><br />' % (url, file))
             scp_push.close()
             self.upload_images(target_folder)
             for image in self.images:
                 if 'kpi-' in image:
                     if '-print' not in image:
-                        text = text + '<img src="%s"></img>' % image
+                        images.append('<img src="%s"></img>' % image)
             self.images = []
 
-            if grafana_token is not None:
-                # get the details of the dashboard through the API, and set the end date to the youngest KPI
-                grafana.list_dashboards()
+            results = csvreader.get_columns(df, ['short-description', 'numeric-score', 'test details', 'test-priority'])
 
-                grafana.create_snapshot(title=grafana_dashboard)
-                time.sleep(3)
-                snapshot = grafana.list_snapshots()[-1]
-                print(snapshot)
-                text = text + '<iframe src="http://%s:3000/dashboard/snapshot/%s" width="100%s" height=1500></iframe><br />' % (
-                    grafana_host, snapshot['key'], '%')
+            low_priority = csvreader.filter_df(results, 'test-priority', 'less than', 94)
+            high_priority = csvreader.filter_df(results, 'test-priority', 'greater than or equal to', 95)
+            high_priority_list.append(high_priority)
 
-            results = csvreader.get_columns(df,['short-description','numeric-score','test details','test-priority'])
-
-            low_priority = csvreader.to_html(csvreader.filter_df(results, 'test-priority', 'less than', 94))
-            high_priority = csvreader.to_html(csvreader.filter_df(results, 'test-priority', 'greater than', 95))
-
-            text = text + 'High priority results: %s' % high_priority
-
-            text = text + 'Low priority results: %s' % low_priority
+            low_priority_list.append(low_priority)
 
         now = date.now()
 
+        high_priority = csvreader.concat(high_priority_list)
+        low_priority = csvreader.concat(low_priority_list)
+
+        high_priority = csvreader.get_columns(high_priority, ['short-description', 'numeric-score', 'test details'])
+        low_priority = csvreader.get_columns(low_priority, ['short-description', 'numeric-score', 'test details'])
+
         if title is None:
             title = "%s %s %s %s:%s report" % (now.day, now.month, now.year, now.hour, now.minute)
 
         # create Grafana Dashboard
         target_files = []
-        for folder in folders:
-            print(folder)
+        for folder in target_folders:
             target_files.append(folder.split('/')[-1] + '/kpi.csv')
+        print('Target files: %s' % target_files)
         grafana.create_custom_dashboard(target_csvs=target_files,
-                                        title=title)
+                                        title=title,
+                                        datasource=grafana_datasource,
+                                        bucket=grafana_bucket)
+
+        try:
+            text = 'Testbed: %s<br />' % testbeds[0]
+        except:
+            text = ''
+
+        for pdf in pdfs:
+            text = text + pdf
+
+        for image in images:
+            text = text + image
+
+        text = text + 'High priority results: %s' % csvreader.to_html(high_priority)
+
+        if grafana_token is not None:
+            # get the details of the dashboard through the API, and set the end date to the youngest KPI
+            grafana.list_dashboards()
+
+            grafana.create_snapshot(title='Testbed: ' + title)
+            time.sleep(3)
+            snapshot = grafana.list_snapshots()[-1]
+            text = text + '<iframe src="http://%s:3000/dashboard/snapshot/%s" width="100%s" height=1500></iframe><br />' % (
+                grafana_host, snapshot['key'], '%')
+
+        text = text + 'Low priority results: %s' % csvreader.to_html(low_priority)
 
         self.create_post(title=title,
                          text=text,
                          tags='custom',
-                         authors=authors)
+                         authors=authors)

py-dashboard/GrafanaRequest.py

@@ -132,6 +132,49 @@ class GrafanaRequest:
         print(dictionary)
         return dictionary
 
+    def maketargets(self,
+                    bucket,
+                    scriptname,
+                    groupBy,
+                    index,
+                    graph_group,
+                    testbed):
+        query = (
+                'from(bucket: "%s")\n  '
+                '|> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  '
+                '|> filter(fn: (r) => r["script"] == "%s")\n   '
+                '|> group(columns: ["_measurement"])\n '
+                % (bucket, scriptname))
+        queryend = ('|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  '
+                    '|> yield(name: "mean")\n  ')
+        if graph_group is not None:
+            graphgroup = ('|> filter(fn: (r) => r["Graph-Group"] == "%s")\n' % graph_group)
+            query += graphgroup
+        if testbed is not None:
+            query += ('|> filter(fn: (r) => r["testbed"] == "%s")\n' % testbed)
+        targets = dict()
+        targets['delimiter'] = ','
+        targets['groupBy'] = groupBy
+        targets['header'] = True
+        targets['ignoreUnknown'] = False
+        targets['orderByTime'] = 'ASC'
+        targets['policy'] = 'default'
+        targets['query'] = query + queryend
+        targets['refId'] = dict(enumerate(string.ascii_uppercase, 1))[index + 1]
+        targets['resultFormat'] = "time_series"
+        targets['schema'] = list()
+        targets['skipRows'] = 0
+        targets['tags'] = list()
+        return targets
+
+    def groupby(self, params, grouptype):
+        dic = dict()
+        dic['params'] = list()
+        dic['params'].append(params)
+        dic['type'] = grouptype
+        return dic
+
+
     def create_custom_dashboard(self,
                                 scripts=None,
                                 title=None,
@@ -174,20 +217,10 @@ class GrafanaRequest:
             target_csvs = open(graph_groups_file).read().split('\n')
             graph_groups = self.get_graph_groups(
                 target_csvs)  # Get the list of graph groups which are in the tests we ran
-            unit_dict = dict()
-            for csv in target_csvs:
-                if len(csv) > 1:
-                    print(csv)
-                    unit_dict.update(self.get_units(csv))
         if target_csvs:
             print('Target CSVs: %s' % target_csvs)
             graph_groups = self.get_graph_groups(
                 target_csvs)  # Get the list of graph groups which are in the tests we ran
-            unit_dict = dict()
-            for csv in target_csvs:
-                if len(csv) > 1:
-                    print(csv)
-                    unit_dict.update(self.get_units(csv))
         for scriptname in graph_groups.keys():
             for graph_group in graph_groups[scriptname]:
                 panel = dict()
@@ -210,14 +243,14 @@ class GrafanaRequest:
                 options = dict()
                 options['alertThreshold'] = True
 
-                #groupBy = list()
-                #groupBy.append(self.groupby('$__interval', 'time'))
-                #groupBy.append(self.groupby('null', 'fill'))
+                groupBy = list()
+                groupBy.append(self.groupby('$__interval', 'time'))
+                groupBy.append(self.groupby('null', 'fill'))
 
-                #targets = list()
-                #counter = 0
-                #new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group, testbed)
-                #targets.append(new_target)
+                targets = list()
+                counter = 0
+                new_target = self.maketargets(bucket, scriptname, groupBy, counter, graph_group, testbed)
+                targets.append(new_target)
 
                 fieldConfig = dict()
                 fieldConfig['defaults'] = dict()
@@ -274,7 +307,7 @@ class GrafanaRequest:
                 panel['spaceLength'] = 10
                 panel['stack'] = False
                 panel['steppedLine'] = False
-                #panel['targets'] = targets
+                panel['targets'] = targets
                 panel['thresholds'] = list()
                 panel['timeFrom'] = None
                 panel['timeRegions'] = list()

py-scripts/ghost_profile.py

@@ -6,17 +6,17 @@ PURPOSE: modify ghost database from the command line.
 SETUP: A Ghost installation which the user has admin access to.
 EXAMPLE: ./ghost_profile.py --article_text_file text.txt --title Test --authors Matthew --ghost_token SECRET_KEY --host 192.168.1.1
 
-There is a specific class for uploading wifi capacity graphs called wifi_capacity.
+There is a specific class for uploading kpi graphs called kpi_to_ghost.
 
 EXAMPLE: ./ghost_profile.py --ghost_token TOKEN --ghost_host 192.168.100.147
 --folders /home/lanforge/html-reports/wifi-capacity-2021-06-04-02-51-07
---wifi_capacity appl --authors Matthew --title 'wifi capacity 2021 06 04 02 51 07' --server 192.168.93.51
+--kpi_to_ghost appl --authors Matthew --title 'wifi capacity 2021 06 04 02 51 07' --server 192.168.93.51
 --user_pull lanforge --password_pull lanforge --customer candela --testbed heather --test_run test-run-6
 --user_push matt --password_push PASSWORD
 
 EXAMPLE 2: ./ghost_profile.py --ghost_token TOKEN
 --ghost_host 192.168.100.147 --server 192.168.93.51 --user_pull lanforge --password_pull lanforge --customer candela
---testbed heather --user_push matt --password_push "amount%coverage;Online" --wifi_capacity app
+--testbed heather --user_push matt --password_push "amount%coverage;Online" --kpi_to_ghost app
 --folders /home/lanforge/html-reports/wifi-capacity-2021-06-14-10-42-29 --grafana_token TOKEN
 --grafana_host 192.168.100.201 --grafana_dashboard 'Stidmatt-02'
 
@@ -60,43 +60,45 @@ class UseGhost(GhostRequest):
         text = open(file).read()
         return self.create_post(title=title, text=text, tags=tags, authors=authors)
 
-    def wifi_capacity(self,
-                      authors,
-                      folders,
-                      title,
-                      server_pull,
-                      ghost_host,
-                      port,
-                      user_pull,
-                      password_pull,
-                      user_push,
-                      password_push,
-                      customer,
-                      testbed,
-                      test_run,
-                      grafana_dashboard,
-                      grafana_token,
-                      grafana_host,
-                      grafana_port):
+    def kpi(self,
+            authors,
+            folders,
+            parent_folder,
+            title,
+            server_pull,
+            ghost_host,
+            port,
+            user_push,
+            password_push,
+            customer,
+            testbed,
+            test_run,
+            grafana_dashboard,
+            grafana_token,
+            grafana_host,
+            grafana_port,
+            datasource,
+            grafana_bucket):
         target_folders = list()
-        return self.wifi_capacity_to_ghost(authors,
-                                           folders,
-                                           title,
-                                           server_pull,
-                                           ghost_host,
-                                           port,
-                                           user_pull,
-                                           password_pull,
-                                           user_push,
-                                           password_push,
-                                           customer,
-                                           testbed,
-                                           test_run,
-                                           target_folders,
-                                           grafana_dashboard,
-                                           grafana_token,
-                                           grafana_host,
-                                           grafana_port)
+        return self.kpi_to_ghost(authors,
+                                 folders,
+                                 parent_folder,
+                                 title,
+                                 server_pull,
+                                 ghost_host,
+                                 port,
+                                 user_push,
+                                 password_push,
+                                 customer,
+                                 testbed,
+                                 test_run,
+                                 target_folders,
+                                 grafana_dashboard,
+                                 grafana_token,
+                                 grafana_host,
+                                 grafana_port,
+                                 datasource,
+                                 grafana_bucket)
 
 
 def main():
@@ -125,12 +127,10 @@ def main():
     optional.add_argument('--image', default=None)
     optional.add_argument('--folder', default=None)
     optional.add_argument('--custom_post', default=None)
-    optional.add_argument('--wifi_capacity', default=None)
+    optional.add_argument('--kpi_to_ghost', help='Generate a Ghost report from KPI spreadsheets', action="store_true")
     optional.add_argument('--folders', action='append', default=None)
     optional.add_argument('--server_pull')
     optional.add_argument('--port', default=22)
-    optional.add_argument('--user_pull', default='lanforge')
-    optional.add_argument('--password_pull', default='lanforge')
     optional.add_argument('--user_push')
     optional.add_argument('--password_push')
     optional.add_argument('--customer')
@@ -140,6 +140,9 @@ def main():
     optional.add_argument('--grafana_token', default=None)
     optional.add_argument('--grafana_host', default=None)
     optional.add_argument('--grafana_port', default=3000)
+    optional.add_argument('--parent_folder', default=None)
+    optional.add_argument('--datasource', default='InfluxDB')
+    optional.add_argument('--grafana_bucket', default=None)
     optional.add_argument('--debug')
     args = parser.parse_args()
 
@@ -165,24 +168,25 @@ def main():
         if args.folder is not None:
             Ghost.upload_images(args.folder)
 
-    if args.wifi_capacity is not None:
-        Ghost.wifi_capacity(args.authors,
-                            args.folders,
-                            args.title,
-                            args.server_pull,
-                            args.ghost_host,
-                            args.port,
-                            args.user_pull,
-                            args.password_pull,
-                            args.user_push,
-                            args.password_push,
-                            args.customer,
-                            args.testbed,
-                            args.test_run,
-                            args.grafana_dashboard,
-                            args.grafana_token,
-                            args.grafana_host,
-                            args.grafana_port)
+    if args.kpi_to_ghost is True:
+        Ghost.kpi(args.authors,
+                  args.folders,
+                  args.parent_folder,
+                  args.title,
+                  args.server_pull,
+                  args.ghost_host,
+                  args.port,
+                  args.user_push,
+                  args.password_push,
+                  args.customer,
+                  args.testbed,
+                  args.test_run,
+                  args.grafana_dashboard,
+                  args.grafana_token,
+                  args.grafana_host,
+                  args.grafana_port,
+                  args.datasource,
+                  args.grafana_bucket)
 
 
 if __name__ == "__main__":

py-scripts/grafana_profile.py

@@ -22,47 +22,6 @@ from LANforge.lfcli_base import LFCliBase
 import string
 
 class UseGrafana(GrafanaRequest):
-    def groupby(self, params, grouptype):
-        dic = dict()
-        dic['params'] = list()
-        dic['params'].append(params)
-        dic['type'] = grouptype
-        return dic
-
-    def maketargets(self,
-                    bucket,
-                    scriptname,
-                    groupBy,
-                    index,
-                    graph_group,
-                    testbed):
-        query = (
-                'from(bucket: "%s")\n  '
-                '|> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  '
-                '|> filter(fn: (r) => r["script"] == "%s")\n   '
-                '|> group(columns: ["_measurement"])\n '
-                % (bucket, scriptname))
-        queryend = ('|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  '
-                    '|> yield(name: "mean")\n  ')
-        if graph_group is not None:
-            graphgroup = ('|> filter(fn: (r) => r["Graph-Group"] == "%s")\n' % graph_group)
-            query += graphgroup
-        if testbed is not None:
-            query += ('|> filter(fn: (r) => r["testbed"] == "%s")\n' % testbed)
-        targets = dict()
-        targets['delimiter'] = ','
-        targets['groupBy'] = groupBy
-        targets['header'] = True
-        targets['ignoreUnknown'] = False
-        targets['orderByTime'] = 'ASC'
-        targets['policy'] = 'default'
-        targets['query'] = query + queryend
-        targets['refId'] = dict(enumerate(string.ascii_uppercase, 1))[index + 1]
-        targets['resultFormat'] = "time_series"
-        targets['schema'] = list()
-        targets['skipRows'] = 0
-        targets['tags'] = list()
-        return targets
 
 
     def read_csv(self, file):