From 062cf04d9be646fca6aa357df37102478a08358d Mon Sep 17 00:00:00 2001 From: Jed Reynolds Date: Fri, 18 Dec 2020 12:36:26 -0800 Subject: [PATCH] lfcli_base.py: adds proxy construction logic --- py-json/LANforge/LFRequest.py | 145 +++++++++++++++++++++------------ py-json/LANforge/lfcli_base.py | 115 +++++++++++++++++++------- 2 files changed, 179 insertions(+), 81 deletions(-) diff --git a/py-json/LANforge/LFRequest.py b/py-json/LANforge/LFRequest.py index 99761ea5..3baed36c 100644 --- a/py-json/LANforge/LFRequest.py +++ b/py-json/LANforge/LFRequest.py @@ -8,9 +8,12 @@ if sys.version_info[0] != 3: exit() import pprint -import urllib.request -import urllib.error -import urllib.parse +import urllib +import time +from urllib import request +from urllib import error +from urllib import parse + import json from LANforge import LFUtils @@ -33,11 +36,27 @@ class LFRequest: # please see this discussion on ProxyHandlers: # https://docs.python.org/3/library/urllib.request.html#urllib.request.ProxyHandler - if proxies_ is not None: - # check to see if proxy has some fields - if ("host" not in proxies_) or ("user" not in proxies_): - raise ValueError("HTTP proxy requires, host, user, pass values.") - self.proxies = proxies_; + # but this makes much more sense: + # https://gist.github.com/aleiphoenix/4159510 + + # if debug_: + # if proxies_ is None: + # print("LFRequest_init_: no proxies_") + # else: + # print("LFRequest: proxies_: ") + # pprint.pprint(proxies_) + + if (proxies_ is not None) and (len(proxies_) > 0): + if ("http" not in proxies_) and ("https" not in proxies_): + raise ValueError("Neither http or https set in proxy definitions. Expects proxy={'http':, 'https':, }") + self.proxies = proxies_ + + # if debug_: + # if self.proxies is None: + # print("LFRequest_init_: no proxies") + # else: + # print("LFRequest: proxies: ") + # pprint.pprint(self.proxies) if not url.startswith("http://") and not url.startswith("https://"): print("No http:// or https:// found, prepending http:// to "+url) @@ -50,7 +69,7 @@ class LFRequest: self.requested_url = url if self.requested_url is None: - raise Exception("Bad LFRequest of url[%s] uri[%s] -> None" % url, uri) + raise Exception("Bad LFRequest of url[%s] uri[%s] -> None" % (url, uri)) if self.requested_url.find('//'): protopos = self.requested_url.find("://") @@ -65,14 +84,6 @@ class LFRequest: if self.debug: print("new LFRequest[%s]" % self.requested_url ) - def update_proxies(self, request): - if (request is None) or (self.proxies is None): - return - - for (proto, host) in self.proxies.items(): - request.set_proxy(host, proto) - - # request first url on stack def formPost(self, show_error=True, debug=False, die_on_error_=False): return self.form_post(show_error=show_error, debug=debug, die_on_error_=die_on_error_) @@ -84,6 +95,13 @@ class LFRequest: debug = True responses = [] urlenc_data = "" + # https://stackoverflow.com/a/59635684/11014343 + if (self.proxies is not None) and (len(self.proxies) > 0): + # https://stackoverflow.com/a/59635684/11014343 + opener = request.build_opener(request.ProxyHandler(self.proxies)) + request.install_opener(opener) + + if (debug): print("formPost: url: "+self.requested_url) if ((self.post_data != None) and (self.post_data is not self.No_Data)): @@ -91,32 +109,31 @@ class LFRequest: if (debug): print("formPost: data looks like:" + str(urlenc_data)) print("formPost: url: "+self.requested_url) - request = urllib.request.Request(url=self.requested_url, - data=urlenc_data, - headers=self.default_headers) + myrequest = request.Request(url=self.requested_url, + data=urlenc_data, + headers=self.default_headers) else: - request = urllib.request.Request(url=self.requested_url, headers=self.default_headers) + myrequest = request.Request(url=self.requested_url, headers=self.default_headers) print("No data for this formPost?") - self.update_proxies(request) + myrequest.headers['Content-type'] = 'application/x-www-form-urlencoded' - request.headers['Content-type'] = 'application/x-www-form-urlencoded' resp = '' try: - resp = urllib.request.urlopen(request) + resp = urllib.request.urlopen(myrequest) responses.append(resp) return responses[0] except urllib.error.HTTPError as error: if (show_error): print("----- LFRequest::formPost:76 HTTPError: --------------------------------------------") - print("%s: %s; URL: %s"%(error.code, error.reason, request.get_full_url())) + print("%s: %s; URL: %s"%(error.code, error.reason, myrequest.get_full_url())) LFUtils.debug_printer.pprint(error.headers) #print("Error: ", sys.exc_info()[0]) #print("Request URL:", request.get_full_url()) - print("Request Content-type:", request.get_header('Content-type')) - print("Request Accept:", request.get_header('Accept')) + print("Request Content-type:", myrequest.get_header('Content-type')) + print("Request Accept:", myrequest.get_header('Accept')) print("Request Data:") - LFUtils.debug_printer.pprint(request.data) + LFUtils.debug_printer.pprint(myrequest.data) if (len(responses) > 0): print("----- Response: --------------------------------------------------------") LFUtils.debug_printer.pprint(responses[0].reason) @@ -127,7 +144,7 @@ class LFRequest: except urllib.error.URLError as uerror: if show_error: print("----- LFRequest::formPost:94 URLError: ---------------------------------------------") - print("Reason: %s; URL: %s"%(uerror.reason, request.get_full_url())) + print("Reason: %s; URL: %s"%(uerror.reason, myrequest.get_full_url())) print("------------------------------------------------------------------------") if (die_on_error_ == True) or (self.die_on_error == True): exit(1) @@ -142,18 +159,25 @@ class LFRequest: if self.die_on_error: die_on_error_ = True responses = [] + if (self.proxies is not None) and (len(self.proxies) > 0): + opener = request.build_opener(request.ProxyHandler(self.proxies)) + request.install_opener(opener) + if ((self.post_data != None) and (self.post_data is not self.No_Data)): - request = urllib.request.Request(url=self.requested_url, - method=method_, - data=json.dumps(self.post_data).encode("utf-8"), - headers=self.default_headers) + myrequest = request.Request(url=self.requested_url, + method=method_, + data=json.dumps(self.post_data).encode("utf-8"), + headers=self.default_headers) else: - request = urllib.request.Request(url=self.requested_url, headers=self.default_headers) + myrequest = request.Request(url=self.requested_url, headers=self.default_headers) print("No data for this jsonPost?") - request.headers['Content-type'] = 'application/json' + myrequest.headers['Content-type'] = 'application/json' + + # https://stackoverflow.com/a/59635684/11014343 + try: - resp = urllib.request.urlopen(request) + resp = request.urlopen(myrequest) resp_data = resp.read().decode('utf-8') if (debug): print("----- LFRequest::json_post:128 debug: --------------------------------------------") @@ -176,14 +200,14 @@ class LFRequest: except urllib.error.HTTPError as error: if show_error or die_on_error_ or (error.code != 404): print("----- LFRequest::json_post:147 HTTPError: --------------------------------------------") - print("<%s> HTTP %s: %s" % (request.get_full_url(), error.code, error.reason )) + print("<%s> HTTP %s: %s" % (myrequest.get_full_url(), error.code, error.reason )) print("Error: ", sys.exc_info()[0]) - print("Request URL:", request.get_full_url()) - print("Request Content-type:", request.get_header('Content-type')) - print("Request Accept:", request.get_header('Accept')) + print("Request URL:", myrequest.get_full_url()) + print("Request Content-type:", myrequest.get_header('Content-type')) + print("Request Accept:", myrequest.get_header('Accept')) print("Request Data:") - LFUtils.debug_printer.pprint(request.data) + LFUtils.debug_printer.pprint(myrequest.data) if error.headers: # the HTTPError is of type HTTPMessage a subclass of email.message @@ -200,7 +224,7 @@ class LFRequest: except urllib.error.URLError as uerror: if show_error: print("----- LFRequest::json_post:171 URLError: ---------------------------------------------") - print("Reason: %s; URL: %s"%(uerror.reason, request.get_full_url())) + print("Reason: %s; URL: %s"%(uerror.reason, myrequest.get_full_url())) print("------------------------------------------------------------------------") if (die_on_error_ == True) or (self.die_on_error == True): exit(1) @@ -225,13 +249,21 @@ class LFRequest: die_on_error_ = True if debug: print("LFUtils.get: url: "+self.requested_url) - myrequest = urllib.request.Request(url=self.requested_url, - headers=self.default_headers, - method=method_) - self.update_proxies(myrequest) + + # https://stackoverflow.com/a/59635684/11014343 + if (self.proxies is not None) and (len(self.proxies) > 0): + opener = request.build_opener(request.ProxyHandler(self.proxies)) + #opener = urllib.request.build_opener(myrequest.ProxyHandler(self.proxies)) + request.install_opener(opener) + + myrequest = request.Request(url=self.requested_url, + headers=self.default_headers, + method=method_) + + myresponses = [] try: - myresponses.append(urllib.request.urlopen(myrequest)) + myresponses.append(request.urlopen(myrequest)) return myresponses[0] except urllib.error.HTTPError as error: if debug: @@ -290,12 +322,25 @@ class LFRequest: self.post_data = data -def plain_get(url_=None, debug_=False, die_on_error_=False): - myrequest = urllib.request.Request(url=url_) +def plain_get(url_=None, debug_=False, die_on_error_=False, proxies_=None): + """ + This static method does not respect LFRequest.proxy, it is not set in scope here + :param url_: + :param debug_: + :param die_on_error_: + :return: + """ + myrequest = request.Request(url=url_) myresponses = [] try: - myresponses.append(urllib.request.urlopen(myrequest)) + if (proxies_ is not None) and (len(proxies_) > 0): + # https://stackoverflow.com/a/59635684/11014343 + opener = myrequest.build_opener(myrequest.ProxyHandler(proxies_)) + myrequest.install_opener(opener) + + myresponses.append(request.urlopen(myrequest)) return myresponses[0] + except urllib.error.HTTPError as error: if debug_: print("----- LFRequest::get:181 HTTPError: --------------------------------------------") diff --git a/py-json/LANforge/lfcli_base.py b/py-json/LANforge/lfcli_base.py index 1bac0034..ed8901ba 100644 --- a/py-json/LANforge/lfcli_base.py +++ b/py-json/LANforge/lfcli_base.py @@ -27,15 +27,23 @@ class LFCliBase: _exit_on_error=False, _exit_on_fail=False, _local_realm=None, + _proxy_str=None, _capture_signal_list=[]): self.fail_pref = "FAILED: " self.pass_pref = "PASSED: " self.lfclient_host = _lfjson_host self.lfclient_port = _lfjson_port self.debug = _debug + # if (_debug): + # print("LFCliBase._proxy_str: %s" % _proxy_str) + self.proxy = {} + self.adjust_proxy(_proxy_str) + if (_local_realm is not None): self.local_realm = _local_realm + # if (_debug): + # print("LFCliBase._proxy_str: %s" % _proxy_str) self.lfclient_url = "http://%s:%s" % (self.lfclient_host, self.lfclient_port) self.test_results = [] self.halt_on_error = _halt_on_error @@ -144,7 +152,6 @@ class LFCliBase: else: if self.debug: print("subclass ignored signal") - def clear_test_results(self): self.test_results.clear() @@ -160,8 +167,13 @@ class LFCliBase: :return: http response object """ json_response = None + debug_ |= self.debug try: - lf_r = LFRequest.LFRequest(self.lfclient_url, _req_url, debug_=self.debug, die_on_error_=self.exit_on_error) + lf_r = LFRequest.LFRequest(url=self.lfclient_url, + uri=_req_url, + proxies_=self.proxy, + debug_=debug_, + die_on_error_=self.exit_on_error) if suppress_related_commands_ is None: if 'suppress_preexec_cli' in _data: del _data['suppress_preexec_cli'] @@ -183,16 +195,16 @@ class LFCliBase: _data['suppress_postexec_method'] = True lf_r.addPostData(_data) - if debug_ or self.debug: + if debug_: LANforge.LFUtils.debug_printer.pprint(_data) - json_response = lf_r.jsonPost(show_error=self.debug, - debug=(self.debug or debug_), + json_response = lf_r.json_post(show_error=debug_, + debug=debug_, response_json_list_=response_json_list_, die_on_error_=self.exit_on_error) if debug_ and (response_json_list_ is not None): pprint.pprint(response_json_list_) except Exception as x: - if self.debug or self.halt_on_error or self.exit_on_error: + if debug_ or self.halt_on_error or self.exit_on_error: print("json_post posted to %s" % _req_url) pprint.pprint(_data) print("Exception %s:" % x) @@ -212,20 +224,25 @@ class LFCliBase: :param response_json_list_: array for json results in the response object, (alternative return method) :return: http response object """ + debug_ |= self.debug json_response = None try: - lf_r = LFRequest.LFRequest(self.lfclient_url, _req_url, debug_=self.debug, die_on_error_=self.exit_on_error) + lf_r = LFRequest.LFRequest(url=self.lfclient_url, + uri=_req_url, + proxies_=self.proxy, + debug_=debug_, + die_on_error_=self.exit_on_error) lf_r.addPostData(_data) - if debug_ or self.debug: + if debug_: LANforge.LFUtils.debug_printer.pprint(_data) json_response = lf_r.json_put(show_error=self.debug, - debug=(self.debug or debug_), + debug=debug_, response_json_list_=response_json_list_, die_on_error_=self.exit_on_error) if debug_ and (response_json_list_ is not None): pprint.pprint(response_json_list_) except Exception as x: - if self.debug or self.halt_on_error or self.exit_on_error: + if debug_ or self.halt_on_error or self.exit_on_error: print("json_put submitted to %s" % _req_url) pprint.pprint(_data) print("Exception %s:" % x) @@ -235,19 +252,26 @@ class LFCliBase: return json_response def json_get(self, _req_url, debug_=False): - if self.debug or debug_: - print("GET: "+_req_url) + debug_ |= self.debug + # if debug_: + # print("json_get: "+_req_url) + # print("json_get: proxies:") + # pprint.pprint(self.proxy) json_response = None # print("----- GET ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ") try: - lf_r = LFRequest.LFRequest(self.lfclient_url, _req_url, debug_=(self.debug or debug_), die_on_error_=self.exit_on_error) - json_response = lf_r.get_as_json(debug_=self.debug, die_on_error_=self.halt_on_error) + lf_r = LFRequest.LFRequest(url=self.lfclient_url, + uri=_req_url, + proxies_=self.proxy, + debug_=debug_, + die_on_error_=self.exit_on_error) + json_response = lf_r.get_as_json(debug_=debug_, die_on_error_=self.halt_on_error) #debug_printer.pprint(json_response) - if (json_response is None) and (self.debug or debug_): + if (json_response is None) and debug_: print("LFCliBase.json_get: no entity/response, probabily status 404") return None except ValueError as ve: - if self.debug or self.halt_on_error or self.exit_on_error: + if debug_ or self.halt_on_error or self.exit_on_error: print("jsonGet asked for " + _req_url) print("Exception %s:" % ve) traceback.print_exception(ValueError, ve, ve.__traceback__, chain=True) @@ -257,22 +281,24 @@ class LFCliBase: return json_response def json_delete(self, _req_url, debug_=False): - if self.debug or debug_: + debug_ |= self.debug + if debug_: print("DELETE: "+_req_url) json_response = None try: # print("----- DELETE ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ") - lf_r = LFRequest.LFRequest(self.lfclient_url, _req_url, - debug_=(self.debug or debug_), + lf_r = LFRequest.LFRequest(url=self.lfclient_url, + uri=_req_url, + proxies_=self.proxy, + debug_=debug_, die_on_error_=self.exit_on_error) - json_response = lf_r.json_delete(debug=self.debug, - die_on_error_=self.halt_on_error) + json_response = lf_r.json_delete(debug=debug_, die_on_error_=self.halt_on_error) #debug_printer.pprint(json_response) - if (json_response is None) and (self.debug or debug_): + if (json_response is None) and debug_: print("LFCliBase.json_delete: no entity/response, probabily status 404") return None except ValueError as ve: - if self.debug or self.halt_on_error or self.exit_on_error: + if debug_ or self.halt_on_error or self.exit_on_error: print("json_delete asked for " + _req_url) print("Exception %s:" % ve) traceback.print_exception(ValueError, ve, ve.__traceback__, chain=True) @@ -356,17 +382,14 @@ class LFCliBase: pass_list.append(result) return pass_list - def get_pass_message(self): pass_messages = self.get_passed_result_list() return "\n".join(pass_messages) - def get_fail_message(self): fail_messages = self.get_failed_result_list() return "\n".join(fail_messages) - def get_all_message(self): return "\n".join(self.test_results) @@ -390,7 +413,6 @@ class LFCliBase: print(message %(fail_len,total_len)) sys.exit(1) - # use this inside the class to log a failure result and print it if wished def _fail(self, message, print_=False): self.test_results.append(self.fail_pref + message) @@ -406,13 +428,38 @@ class LFCliBase: print(message %(num_passing,num_total)) sys.exit(0) - # use this inside the class to log a pass result and print if wished. def _pass(self, message, print_=False): self.test_results.append(self.pass_pref + message) if print_: print(self.pass_pref + message) - #Create argparse with radio, securiy, ssid and passwd required + + def adjust_proxy(self, proxy_str): + # if self.debug: + # print("lfclibase.adjust_proxy: %s" % proxy_str) + if (proxy_str is None) or (proxy_str == ""): + return + if self.proxy is None: + self.proxy = {} + + if proxy_str.find("http:") > -1: + self.proxy["http"] = proxy_str + if proxy_str.find("https:") > -1: + self.proxy["https"] = proxy_str + # if self.debug: + # print("lfclibase::self.proxy: ") + # pprint.pprint(self.proxy) + + + # This style of Action subclass for argparse can't do much unless we incorporate + # our argparse as a member of LFCliBase. Then we can do something like automatically + # parse our proxy string without using _init_ arguments + # class ProxyAction(argparse.Action, zelf): + # def __init__(self, outter_): + # pass + # def __call__(self, parser, namespace, values, option_string=None): + # zelf.adjust_proxy(values) + @staticmethod def create_bare_argparse(prog=None, formatter_class=None, epilog=None, description=None): if (prog is not None) or (formatter_class is not None) or (epilog is not None) or (description is not None): @@ -428,13 +475,18 @@ class LFCliBase: optional.add_argument('--mgr', help='hostname for where LANforge GUI is running', default='localhost') optional.add_argument('--mgr_port', help='port LANforge GUI HTTP service is running on', default=8080) optional.add_argument('--debug', help='Enable debugging', default=False, action="store_true") + optional.add_argument('--proxy', nargs='?', default=None, # action=ProxyAction, + help='Connection proxy like http://proxy.localnet:80 or https://user:pass@proxy.localnet:3128') return parser # Create argparse with radio, securiy, ssid and passwd required # TODO: show example of how to add required or optional arguments from calling class @staticmethod - def create_basic_argparse(prog=None, formatter_class=None, epilog=None, description=None): + def create_basic_argparse(prog=None, + formatter_class=None, + epilog=None, + description=None): if (prog is not None) or (formatter_class is not None) or (epilog is not None) or (description is not None): parser = argparse.ArgumentParser(prog=prog, formatter_class=formatter_class, @@ -442,7 +494,6 @@ class LFCliBase: description=description) else: parser = argparse.ArgumentParser() - optional = parser.add_argument_group('optional arguments') required = parser.add_argument_group('required arguments') #Optional Args @@ -454,6 +505,8 @@ class LFCliBase: optional.add_argument('--num_stations', help='Number of stations to create', default=0) optional.add_argument('--test_id', help='Test ID (intended to use for ws events)', default="webconsole") optional.add_argument('--debug', help='Enable debugging', default=False, action="store_true") + optional.add_argument('--proxy', nargs='?', default=None, + help='Connection proxy like http://proxy.localnet:80 or https://user:pass@proxy.localnet:3128') #Required Args required.add_argument('--radio', help='radio EID, e.g: 1.wiphy2', required=True) required.add_argument('--security', help='WiFi Security protocol: < open | wep | wpa | wpa2 | wpa3 >', required=True)