json: Add methods to wait for cx, endp to be found.

This lets scripts wait until we are sure the system can handle
further work.

Ensure we print out error in case the URL processing logic
detects and error and plans to exit.

py-json/LANforge/LFRequest.py

@@ -137,7 +137,7 @@ class LFRequest:
                 response_json_list_.append(j)
             return responses[0]
         except urllib.error.HTTPError as error:
-            if show_error:
+            if show_error or die_on_error_ or (error.code != 404):
                 print("----- LFRequest::jsonPost:138 HTTPError: --------------------------------------------")
                 print("<%s> HTTP %s: %s"%(request.get_full_url(), error.code, error.reason, ))
 

py-json/LANforge/LFUtils.py

@@ -400,7 +400,6 @@ def wait_until_ports_appear(base_url="http://localhost:8080", port_list=(), debu
     """
     print("Waiting until ports appear...")
     found_stations = []
-    sleep(2)
     port_url = "/port/1"
     ncshow_url = "/cli-form/nc_show_ports"
     if base_url.endswith('/'):
@@ -432,6 +431,47 @@ def wait_until_ports_appear(base_url="http://localhost:8080", port_list=(), debu
         print("These stations appeared: " + ", ".join(found_stations))
     return
 
+def wait_until_endps(base_url="http://localhost:8080", endp_list=(), debug=False):
+    """
+
+    :param base_url:
+    :param port_list:
+    :param debug:
+    :return:
+    """
+    print("Waiting until endpoints appear...")
+    found_endps = []
+    port_url = "/port/1"
+    ncshow_url = "/cli-form/show_endp"
+    if base_url.endswith('/'):
+        port_url = port_url[1:]
+        ncshow_url = ncshow_url[1:]
+
+    while len(found_stations) < len(port_list):
+        found_stations = []
+        for port_eid in port_list:
+
+            eid = name_to_eid(port_eid)
+            shelf = eid[0]
+            resource_id = eid[1]
+            port_name = eid[2]
+            
+            uri = "%s/%s/%s" % (port_url, resource_id, port_name)
+            lf_r = LFRequest.LFRequest(base_url, uri)
+            json_response = lf_r.getAsJson(debug_=False)
+            if (json_response != None):
+                found_stations.append(port_name)
+            else:
+                lf_r = LFRequest.LFRequest(base_url, ncshow_url)
+                lf_r.addPostData({"shelf": shelf, "resource": resource_id, "port": port_name, "flags": 1})
+                lf_r.formPost()
+        if (len(found_stations) < len(port_list)):
+            sleep(2)
+
+    if debug:
+        print("These stations appeared: " + ", ".join(found_stations))
+    return
+
 
 def removePort(resource, port_name, baseurl="http://localhost:8080/", debug=False):
     if debug:

py-json/LANforge/lfcli_base.py

@@ -196,7 +196,7 @@ class LFCliBase:
     # use this inside the class to log a failure result
     def _fail(self, message, print_=False):
         self.test_results.append(self.fail_pref + message)
-        if print_:
+        if print_ or self.exit_on_fail:
             print(self.fail_pref + message)
         if self.exit_on_fail:
             sys.exit(1)

py-json/realm.py

@@ -191,6 +191,53 @@ class Realm(LFCliBase):
         response = super().json_get("/cx")
         return response
 
+    def waitUntilEndpsAppear(self, these_endp):
+        wait_more = True;
+        count = 0
+        while wait_more:
+            wait_more = False
+            endp_list = self.json_get("/endp")
+            found_endps = {}
+            if endp_list is not None:
+                endp_list = list(endp_list['endpoint'])
+                for endp_name in range(len(endp_list)):
+                    name = list(endp_list[endp_name])[0]
+                    found_endps[name] = name
+
+            for req in these_endp:
+                if not req in found_endps:
+                    print("Waiting on endpoint: %s"%(req))
+                    wait_more = True
+            count += 1
+            if (count > 100):
+                break
+
+        return not wait_more
+
+    def waitUntilCxsAppear(self, these_cx):
+        wait_more = True;
+        count = 0
+        while wait_more:
+            wait_more = False
+            found_cxs = {}
+            cx_list = list(self.cx_list())
+            not_cx = ['warnings', 'errors', 'handler', 'uri', 'items']
+            if cx_list is not None:
+                for cx_name in cx_list:
+                    if cx_name in not_cx:
+                        continue
+                    found_cxs[cx_name] = cx_name
+
+            for req in these_cx:
+                if not req in found_cxs:
+                    print("Waiting on CX: %s"%(req))
+                    wait_more = True
+            count += 1
+            if (count > 100):
+                break
+
+        return not wait_more
+
     # Returns map of all stations with port+type == WIFI-STATION
     def station_map(self):
         response = super().json_get("/port/list?fields=_links,alias,device,port+type")
@@ -521,6 +568,7 @@ class MULTICASTProfile(LFCliBase):
             debug_=True
 
         for endp_name in self.get_mc_names():
+            print("Starting mcast endpoint: %s"%(endp_name))
             json_data = {
                 "endp_name":endp_name
             }
@@ -558,7 +606,7 @@ class MULTICASTProfile(LFCliBase):
         side_tx_shelf = side_tx_info[0]
         side_tx_resource = side_tx_info[1]
         side_tx_port = side_tx_info[2]
-        side_tx_name = "mtx-%s-%i-"%(side_tx_port, len(created_mc))
+        side_tx_name = "mtx-%s-%i-"%(side_tx_port, len(self.created_mc))
 
         json_data = []
         
@@ -598,18 +646,24 @@ class MULTICASTProfile(LFCliBase):
         url = "cli-json/set_mc_endp"
         self.local_realm.json_post(url, json_data, debug_=debug_, suppress_related_commands_=suppress_related_commands)
 
-        created_mc[side_tx_name] = side_tx_name
+        self.created_mc[side_tx_name] = side_tx_name
+
+        these_endp = [side_tx_name]
+        self.local_realm.waitUntilEndpsAppear(these_endp)
+
 
     def create_mc_rx(self, endp_type, side_rx, suppress_related_commands=None, debug_ = False):
         if self.debug:
             debug_=True
 
+        these_endp = []
+
         for port_name in side_rx:
             side_rx_info = self.local_realm.name_to_eid(port_name)
             side_rx_shelf = side_rx_info[0]
             side_rx_resource = side_rx_info[1]
             side_rx_port = side_rx_info[2]
-            side_rx_name = "mrx-%s-%i-"%(side_tx_port, len(created_mc))
+            side_rx_name = "mrx-%s-%i-"%(side_rx_port, len(self.created_mc))
             # add_endp mcast-rcv-sta-001 1 1 sta0002 mc_udp 9999 NO 0 0 NO 1472 0 INCREASING NO 32 0 0
             json_data = {
                             'alias':side_rx_name,
@@ -644,8 +698,10 @@ class MULTICASTProfile(LFCliBase):
             url = "cli-json/set_mc_endp"
             self.local_realm.json_post(url, json_data, debug_=debug_, suppress_related_commands_=suppress_related_commands)
 
-            created_mc[side_rx_name] = side_rx_name
+            self.created_mc[side_rx_name] = side_rx_name
+            these_endp.append(side_rx_name)
 
+        self.local_realm.waitUntilEndpsAppear(these_endp)
 
     def to_string(self):
         pprint.pprint(self)
@@ -732,6 +788,7 @@ class L3CXProfile(LFCliBase):
         print("Cleaning up cxs and endpoints")
         if len(self.created_cx) != 0:
             for cx_name in self.created_cx.keys():
+                print("Cleaning cx: %s"%(cx_name))
                 req_url = "cli-json/rm_cx"
                 data = {
                     "test_mgr": "default_tm",
@@ -740,9 +797,11 @@ class L3CXProfile(LFCliBase):
                 self.json_post(req_url, data)
 
                 for side in range(len(self.created_cx[cx_name])):
+                    ename = self.created_cx[cx_name][side]
+                    print("Cleaning endpoint: %s"%(ename))
                     req_url = "cli-json/rm_endp"
                     data = {
-                        "endp_name": self.created_cx[cx_name][side]
+                        "endp_name": ename
                     }
                     self.json_post(req_url, data)
 
@@ -752,6 +811,9 @@ class L3CXProfile(LFCliBase):
 
         cx_post_data = []
         timer_post_data = []
+        these_endp = []
+        these_cx = []
+
         # print(self.side_a_min_rate, self.side_a_max_rate)
         # print(self.side_b_min_rate, self.side_b_max_rate)
         if (self.side_a_min_bps is None) \
@@ -779,6 +841,9 @@ class L3CXProfile(LFCliBase):
                 self.created_cx[ cx_name ] = [endp_a_name, endp_b_name]
                 self.created_endp[endp_a_name] = endp_a_name;
                 self.created_endp[endp_b_name] = endp_b_name;
+                these_cx.append(cx_name)
+                these_endp.append(endp_a_name)
+                these_endp.append(endp_b_name)
                 endp_side_a = {
                     "alias": endp_a_name,
                     "shelf": side_a_shelf,
@@ -860,6 +925,9 @@ class L3CXProfile(LFCliBase):
                 self.created_cx[ cx_name ] = [endp_a_name, endp_b_name]
                 self.created_endp[endp_a_name] = endp_a_name;
                 self.created_endp[endp_b_name] = endp_b_name;
+                these_cx.append(cx_name)
+                these_endp.append(endp_a_name)
+                these_endp.append(endp_b_name)
                 endp_side_a = {
                     "alias": endp_a_name,
                     "shelf": side_a_shelf,
@@ -927,9 +995,12 @@ class L3CXProfile(LFCliBase):
             url = "/cli-json/add_cx"
             self.local_realm.json_post(url, data, debug_=debug_, suppress_related_commands_=suppress_related_commands)
             #print(" napping %f sec"%sleep_time, end='')
-            time.sleep(sleep_time)
+            #time.sleep(sleep_time)
         #print("")
 
+        self.local_realm.waitUntilEndpsAppear(these_endp)
+        self.local_realm.waitUntilCxsAppear(these_cx)
+
     def to_string(self):
         pprint.pprint(self)
 
@@ -1772,19 +1843,21 @@ class StationProfile:
             "port": None
         }
         if (desired_stations is None):
-            return
+            desired_stations = self.station_names;
+
         if len(desired_stations) < 1:
+            print("ERROR:  StationProfile cleanup, list is empty")
             return
 
         del_count = len(desired_stations)
 
         # First, request remove on the list.
         for port_eid in desired_stations:
-            eid = self.name_to_eid(port_eid)
+            eid = self.local_realm.name_to_eid(port_eid)
             data["shelf"] = eid[0]
             data["resource"] = eid[1]
             data["port"] = eid[2]
-            self.local_realm.json_post(req_url, data, debug_=self.debug)
+            self.local_realm.json_post(req_url, data, debug_=True) #self.debug)
             time.sleep(delay)
 
         # And now see if they are gone
@@ -1792,7 +1865,7 @@ class StationProfile:
         while count < (del_count + 10):
             found_one = False
             for port_eid in desired_stations:
-                eid = self.name_to_eid(port_eid)
+                eid = self.local_realm.name_to_eid(port_eid)
                 data["shelf"] = eid[0]
                 data["resource"] = eid[1]
                 data["port"] = eid[2]
@@ -1803,7 +1876,7 @@ class StationProfile:
                     data["shelf"] = eid[0]
                     data["resource"] = eid[1]
                     data["port"] = eid[2]
-                    self.local_realm.json_post(req_url, data, debug_=self.debug)
+                    self.local_realm.json_post(req_url, data, debug_=True) #self.debug)
                     time.sleep(delay)
             if not found_one:
                 return

py-scripts/test_l3_longevity.py

@@ -124,10 +124,13 @@ class L3VariableTimeLongevity(LFCliBase):
         else:
             print("print failed to get IP's")
 
-        self.multicast_profile.start_mc()
+        print("Starting multicast traffic (if any configured)")
+        self.multicast_profile.start_mc(debug_=True)
+        print("Starting layer-3 traffic (if any configured)")
         self.cx_profile.start_cx()
 
         cur_time = datetime.datetime.now()
+        print("Getting initial values.")
         old_rx_values = self.__get_rx_values()
 
         end_time = self.local_realm.parse_time(self.test_duration) + cur_time
@@ -191,21 +194,23 @@ class L3VariableTimeLongevity(LFCliBase):
                 index += 1
 
             for etype in self.endp_types:
+                print("Creating connections for endpoint type: %s"%(etype))
                 if etype == "mc_udp" or etype == "mc_udp6":
                     self.multicast_profile.create_mc_tx(etype, self.side_b, etype)
-                    self.multicast_profile.create_mc_rx(etype, side_rx=station_profle.station_names)
+                    self.multicast_profile.create_mc_rx(etype, side_rx=station_profile.station_names)
                 else:
                     self.cx_profile.create(endp_type=etype, side_a=station_profile.station_names, side_b=self.side_b, sleep_time=0)
 
         self._pass("PASS: Stations build finished")
 
-def valid_endp_types(endp_type):
-    etypes = endp_type.split()
+def valid_endp_types(_endp_type):
+    etypes = _endp_type.split()
     for endp_type in etypes:
         valid_endp_type=['lf_udp','lf_udp6','lf_tcp','lf_tcp6','mc_udp','mc_udp6']
         if not (str(endp_type) in valid_endp_type):
             print('invalid endp_type: %s. Valid types lf_udp, lf_udp6, lf_tcp, lf_tcp6, mc_udp, mc_udp6' % endp_type)
             exit(1)
+    return _endp_type
 
 def main():
     lfjson_host = "localhost"
@@ -336,6 +341,8 @@ Note:   multiple --radio switches may be entered up to the number of radios avai
         station_lists.append(station_list)
         index += 1
 
+    print("endp-types: %s"%(endp_types))
+
     ip_var_test = L3VariableTimeLongevity(lfjson_host, 
                                    lfjson_port, 
                                    number_template="00",