From 9e51313b4ff7b7387eb3b59641eac8cfc7122d56 Mon Sep 17 00:00:00 2001 From: Hoang Hong Quan Date: Sun, 23 Mar 2025 01:20:22 +0700 Subject: [PATCH] Improved error handling, gzip and deflate support, and download progress display --- Scripts/resource_fetcher.py | 139 ++++++++++++++++++++++++++++-------- 1 file changed, 109 insertions(+), 30 deletions(-) diff --git a/Scripts/resource_fetcher.py b/Scripts/resource_fetcher.py index 7c900ea..c853098 100644 --- a/Scripts/resource_fetcher.py +++ b/Scripts/resource_fetcher.py @@ -4,17 +4,21 @@ import json import plistlib import socket import sys +import gzip +import zlib +import time if sys.version_info >= (3, 0): from urllib.request import urlopen, Request + from urllib.error import URLError else: import urllib2 - from urllib2 import urlopen, Request + from urllib2 import urlopen, Request, URLError class ResourceFetcher: def __init__(self, headers=None): self.request_headers = headers or { - "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36' + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" } self.buffer_size = 16 * 1024 self.ssl_context = self.create_ssl_context() @@ -27,65 +31,140 @@ class ResourceFetcher: cafile = certifi.where() ssl_context = ssl.create_default_context(cafile=cafile) except Exception as e: - print("SSL Context Creation Error: {}".format(e)) + print("Failed to create SSL context: {}".format(e)) ssl_context = ssl._create_unverified_context() return ssl_context - def is_connected(self, timeout=5): - socket.create_connection(("github.com", 443), timeout=timeout) - - def _make_request(self, resource_url): - self.is_connected() - + def _make_request(self, resource_url, timeout=10): try: - return urlopen(Request(resource_url, headers=self.request_headers), context=self.ssl_context) + headers = dict(self.request_headers) + headers["Accept-Encoding"] = "gzip, deflate" + + return urlopen(Request(resource_url, headers=headers), timeout=timeout, context=self.ssl_context) + except socket.timeout as e: + print("Timeout error: {}".format(e)) + except ssl.SSLError as e: + print("SSL error: {}".format(e)) + except (URLError, socket.gaierror) as e: + print("Connection error: {}".format(e)) except Exception as e: - pass + print("Request failed: {}".format(e)) return None def fetch_and_parse_content(self, resource_url, content_type=None): - response = self._make_request(resource_url) + attempt = 0 + response = None + + while attempt < 3: + response = self._make_request(resource_url) + + if not response: + attempt += 1 + print("Failed to fetch content from {}. Retrying...".format(resource_url)) + continue + + if response.getcode() == 200: + break + + attempt += 1 if not response: + print("Failed to fetch content from {}".format(resource_url)) return None content = response.read() - if content_type == 'json': - return json.loads(content) - elif content_type == 'plist': - return plistlib.loads(content) - else: - return content.decode('utf-8') + + if response.info().get("Content-Encoding") == "gzip" or content.startswith(b"\x1f\x8b"): + try: + content = gzip.decompress(content) + except Exception as e: + print("Failed to decompress gzip content: {}".format(e)) + elif response.info().get("Content-Encoding") == "deflate": + try: + content = zlib.decompress(content) + except Exception as e: + print("Failed to decompress deflate content: {}".format(e)) + + try: + if content_type == "json": + return json.loads(content) + elif content_type == "plist": + return plistlib.loads(content) + else: + return content.decode("utf-8") + except Exception as e: + print("Error parsing content as {}: {}".format(content_type, e)) + + return None def _download_with_progress(self, response, local_file): - total_size = response.getheader('Content-Length') + total_size = response.getheader("Content-Length") if total_size: total_size = int(total_size) bytes_downloaded = 0 + start_time = time.time() + last_time = start_time + last_bytes = 0 + speeds = [] + speed_str = "-- KB/s" + while True: chunk = response.read(self.buffer_size) if not chunk: break local_file.write(chunk) bytes_downloaded += len(chunk) - + + current_time = time.time() + time_diff = current_time - last_time + + if time_diff > 0.5: + current_speed = (bytes_downloaded - last_bytes) / time_diff + speeds.append(current_speed) + if len(speeds) > 5: + speeds.pop(0) + avg_speed = sum(speeds) / len(speeds) + + if avg_speed < 1024*1024: + speed_str = "{:.1f} KB/s".format(avg_speed/1024) + else: + speed_str = "{:.1f} MB/s".format(avg_speed/(1024*1024)) + + last_time = current_time + last_bytes = bytes_downloaded + if total_size: percent = int(bytes_downloaded / total_size * 100) - progress = f"[{'=' * (percent // 2):50s}] {percent}% {bytes_downloaded / (1024 * 1024):.2f}/{total_size / (1024 * 1024):.2f} MB" + bar_length = 40 + filled = int(bar_length * bytes_downloaded / total_size) + bar = "█" * filled + "░" * (bar_length - filled) + progress = "{} [{}] {:3d}% {:.1f}/{:.1f}MB".format(speed_str, bar, percent, bytes_downloaded/(1024*1024), total_size/(1024*1024)) else: - progress = f"Downloaded {bytes_downloaded / (1024 * 1024):.2f} MB" - print(progress, end='\r') - + progress = "{} {:.1f}MB downloaded".format(speed_str, bytes_downloaded/(1024*1024)) + + print(" " * 80, end="\r") + print(progress, end="\r") + print() def download_and_save_file(self, resource_url, destination_path): - response = self._make_request(resource_url) + attempt = 0 - if not response: - return None + while attempt < 3: + response = self._make_request(resource_url) - with open(destination_path, 'wb') as local_file: - print(f"Downloading from {resource_url}") - self._download_with_progress(response, local_file) \ No newline at end of file + if not response: + attempt += 1 + print("Failed to download file from {}. Retrying...".format(resource_url)) + continue + + self._download_with_progress(response, open(destination_path, "wb")) + + if os.path.exists(destination_path) and os.path.getsize(destination_path) > 0: + return True + + attempt += 1 + + return False \ No newline at end of file