start of monitor additions & urls_per_ten remodeling

py-json/realm.py

@@ -1130,10 +1130,6 @@ class L3CXProfile(BaseProfile):
                 return False
         else:
             return False
-
-    """
-    coloumn names
-    """
     def monitor(self,
                 duration_sec=60,
                 monitor_interval=1,
@@ -1604,6 +1600,23 @@ class L4CXProfile(LFCliBase):
             print(".", end='')
         print("")
 
+    def check_request_rate(self):
+        endp_list = self.json_get("layer4/list?fields=urls/s")
+        expected_passes = 0
+        passes = 0
+        if endp_list is not None and endp_list['endpoint'] is not None:
+            endp_list = endp_list['endpoint']
+            for item in endp_list:
+                for name, info in item.items():
+                    if name in self.cx_profile.created_cx.keys():
+                        expected_passes += 1
+                        if info['urls/s'] * self.requests_per_ten >= self.target_requests_per_ten * .9:
+                            print(name, info['urls/s'], info['urls/s'] * self.requests_per_ten, self.target_requests_per_ten * .9)
+                            passes += 1
+
+        return passes == expected_passes
+
+
     def cleanup(self):
         print("Cleaning up cxs and endpoints")
         if len(self.created_cx) != 0:
@@ -1662,6 +1675,155 @@ class L4CXProfile(LFCliBase):
                                        suppress_related_commands_=suppress_related_commands_)
             time.sleep(sleep_time)
 
+################# MONITOR CODE IN L4CXPROF#######################
+
+
+    def monitor(self,
+                duration_sec=60,
+                monitor_interval=1,
+                col_names=None,
+                created_cx=None,
+                monitor=True,
+                report_file=None,
+                output_format=None,
+                script_name=None,
+                arguments=None,
+                iterations=0):
+        try:
+            duration_sec = self.parse_time(duration_sec).seconds
+        except:
+            if (duration_sec is None) or (duration_sec <= 1):
+                raise ValueError("L4CXProfile::monitor wants duration_sec > 1 second")
+            if (duration_sec <= monitor_interval):
+                raise ValueError("L4CXProfile::monitor wants duration_sec > monitor_interval")
+        if report_file == None:
+            raise ValueError("Monitor requires an output file to be defined")
+        if created_cx == None:
+            raise ValueError("Monitor needs a list of Layer 4 connections")
+        if (monitor_interval is None) or (monitor_interval < 1):
+            raise ValueError("L4CXProfile::monitor wants monitor_interval >= 1 second")
+        if col_names is None:
+            raise ValueError("L4CXProfile::monitor wants a list of column names to monitor")
+        if output_format is not None:
+            if output_format.lower() != report_file.split('.')[-1]:
+                if output_format.lower() != 'excel':
+                    raise ValueError('Filename %s does not match output format %s' % (report_file, output_format))
+        else:
+            output_format = report_file.split('.')[-1]
+
+        # Step 1, column names
+        fields = ",".join(col_names)
+        print(fields)
+        # Step 2, monitor columns, 
+        start_time = datetime.datetime.now()
+        end_time = start_time + datetime.timedelta(seconds=duration_sec)
+        #print(end_time)
+        sleep_interval =  duration_sec // 5
+        value_map = dict()
+        passes = 0
+        expected_passes = 0
+    
+        timestamps = []
+        for x in range(0,int(round(iterations,0))):
+            while datetime.datetime.now() < end_time:
+                response = self.json_get("layer4/list?fields=urls/s")
+                if "endpoint" not in response:
+                    print(response)
+                    raise ValueError("no endpoint?")
+                if monitor:
+                    if self.debug:
+                        print(response)
+                t = datetime.datetime.now()
+                timestamps.append(t)
+                value_map[t] = response
+                if self.debug:
+                    print(old_cx_rx_values, new_cx_rx_values)
+                    print("\n-----------------------------------")
+                    print(curr_time)
+                    print("-----------------------------------\n")
+                expected_passes += 1
+                if self.__compare_vals(old_cx_rx_values, new_cx_rx_values):
+                    passes += 1
+                else:
+                    self._fail("FAIL: Not all stations increased traffic")
+                    self.exit_fail()
+                old_cx_rx_values = new_cx_rx_values
+                time.sleep(monitor_interval)
+        print(value_map)
+
+        # if passes == expected_passes:
+        # self._pass("PASS: All tests passed")
+        # step 3 organize data
+        endpoints = list()
+        for endpoint in value_map.values():
+            endpoints.append(endpoint['endpoint'])
+        endpoints2 = []
+        for y in range(0, len(endpoints)):
+            for x in range(0, len(endpoints[0])):
+                endpoints2.append(list(list(endpoints[y][x].values())[0].values()))
+        import itertools
+        timestamps2 = list(
+            itertools.chain.from_iterable(itertools.repeat(x, len(created_cx.split(','))) for x in timestamps))
+        for point in range(0, len(endpoints2)):
+            endpoints2[point].insert(0, timestamps2[point])
+        # step 4 save and close
+        header_row = col_names
+        header_row.insert(0, 'Timestamp')
+        print(header_row)
+        if output_format.lower() in ['excel', 'xlsx'] or report_file.split('.')[-1] == 'xlsx':
+            report_fh = open(report_file, "w+")
+            workbook = xlsxwriter.Workbook(report_file)
+            worksheet = workbook.add_worksheet()
+            for col_num, data in enumerate(header_row):
+                worksheet.write(0, col_num, data)
+            row_num = 1
+            for x in endpoints2:
+                for col_num, data in enumerate(x):
+                    worksheet.write(row_num, col_num, str(data))
+                row_num += 1
+            workbook.close()
+        else:
+            df = pd.DataFrame(endpoints2)
+            df.columns = header_row
+            import requests
+            import ast
+            try:
+                systeminfo = ast.literal_eval(requests.get('http://localhost:8090').text)
+            except:
+                systeminfo = ast.literal_eval(requests.get('http://localhost:8090').text)
+            df['LFGUI Release'] = systeminfo['VersionInfo']['BuildVersion']
+            df['Script Name'] = script_name
+            df['Arguments'] = arguments
+            for x in ['LFGUI Release', 'Script Name', 'Arguments']:
+                df[x][1:] = ''
+            if output_format == 'pdf':
+                import matplotlib.pyplot as plt
+                from matplotlib.backends.backend_pdf import PdfPages
+                fig, ax = plt.subplots(figsize=(12, 4))
+                ax.axis('tight')
+                ax.axis('off')
+                the_table = ax.table(cellText=df.values, colLabels=df.columns, loc='center')
+                pp = PdfPages(report_file)
+                pp.savefig(fig, bbox_inches='tight')
+                pp.close()
+            if output_format == 'hdf':
+                df.to_hdf(report_file, 'table', append=True)
+            if output_format == 'parquet':
+                df.to_parquet(report_file, engine='pyarrow')
+            if output_format == 'png':
+                fig = df.plot().get_figure()
+                fig.savefig(report_file)
+            if output_format == 'html':
+                print('Shivams function')
+            if output_format == 'df':
+                return df
+            supported_formats = ['csv', 'json', 'stata', 'pickle']
+            for x in supported_formats:
+                if output_format.lower() == x or report_file.split('.')[-1] == x:
+                    exec('df.to_' + x + '("' + report_file + '")')
+            else:
+                pass
+#end of L4CXProf class 
 
 class GenCXProfile(LFCliBase):
     def __init__(self, lfclient_host, lfclient_port, local_realm, debug_=False):

py-scripts/test_ipv4_l4_urls_per_ten.py

@@ -24,6 +24,7 @@ from LANforge import LFUtils
 import realm
 import time
 import datetime
+from realm import TestGroupProfile
 
 
 class IPV4L4(LFCliBase):
@@ -66,21 +67,20 @@ class IPV4L4(LFCliBase):
         self.cx_profile.url = self.url
         self.cx_profile.requests_per_ten = self.requests_per_ten
 
-    def __check_request_rate(self):
-        endp_list = self.json_get("layer4/list?fields=urls/s")
-        expected_passes = 0
-        passes = 0
-        if endp_list is not None and endp_list['endpoint'] is not None:
-            endp_list = endp_list['endpoint']
-            for item in endp_list:
-                for name, info in item.items():
-                    if name in self.cx_profile.created_cx.keys():
-                        expected_passes += 1
-                        if info['urls/s'] * self.requests_per_ten >= self.target_requests_per_ten * .9:
-                            print(name, info['urls/s'], info['urls/s'] * self.requests_per_ten, self.target_requests_per_ten * .9)
-                            passes += 1
-
-        return passes == expected_passes
+  #  def __check_request_rate(self):
+  #      endp_list = self.json_get("layer4/list?fields=urls/s")
+  #      expected_passes = 0
+  #      passes = 0
+  #      if endp_list is not None and endp_list['endpoint'] is not None:
+  #          endp_list = endp_list['endpoint']
+  #          for item in endp_list:
+  #              for name, info in item.items():
+  #                  if name in self.cx_profile.created_cx.keys():
+  #                      expected_passes += 1
+  #                      if info['urls/s'] * self.requests_per_ten >= self.target_requests_per_ten * .9:
+  #                          print(name, info['urls/s'], info['urls/s'] * self.requests_per_ten, self.target_requests_per_ten * .9)
+  #                          passes += 1
+  #      return passes == expected_passes
 
     def build(self):
         # Build stations
@@ -106,9 +106,9 @@ class IPV4L4(LFCliBase):
             exit(1)
         self.cx_profile.start_cx()
         print("Starting test")
-        curr_time = datetime.datetime.now()
+        curr_time = datetime.datetime.now() 
         end_time = self.local_realm.parse_time(self.test_duration) + curr_time
-        sleep_interval = self.local_realm.parse_time(self.test_duration) // 5
+       sleep_interval = self.local_realm.parse_time(self.test_duration) // 5
         passes = 0
         expected_passes = 0
         for test in range(self.num_tests):