fix: replace requests with httpx for improved HTTP handling

Author: KotaHv

backend/requirements.txt

@@ -9,7 +9,7 @@ idna==3.4
 pydantic~=1.10
 PySocks==1.7.1
 qbittorrent-api==2023.9.53
-requests==2.31.0
+httpx[socks,http2]==0.27.0
 six==1.16.0
 sniffio==1.3.0
 soupsieve==2.4.1

backend/src/module/checker/checker.py

@@ -1,7 +1,7 @@
 import logging
 from pathlib import Path
 
-import requests
+import httpx
 from sqlalchemy import inspect
 
 from module.conf import VERSION, settings
@@ -58,7 +58,7 @@ def check_downloader() -> bool:
                 if "://" not in settings.downloader.host
                 else f"{settings.downloader.host}"
             )
-            response = requests.get(url, timeout=2)
+            response = httpx.get(url, timeout=2)
             # if settings.downloader.type in response.text.lower():
             if (
                 "qbittorrent" in response.text.lower()
@@ -71,10 +71,10 @@ def check_downloader() -> bool:
                         return False
             else:
                 return False
-        except requests.exceptions.ReadTimeout:
+        except httpx.ReadTimeout:
             logger.error("[Checker] Downloader connect timeout.")
             return False
-        except requests.exceptions.ConnectionError:
+        except httpx.ConnectError:
             logger.error("[Checker] Downloader connect failed.")
             return False
         except Exception as e:

backend/src/module/conf/log.py

@@ -32,3 +32,6 @@ def setup_logger(level: int = logging.INFO, reset: bool = False):
             logging.StreamHandler(),
         ],
     )
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    logging.getLogger("httpcore").setLevel(logging.WARNING)
+    logging.getLogger("hpack").setLevel(logging.WARNING)

backend/src/module/network/request_url.py

@@ -2,7 +2,7 @@
 import socket
 import time
 
-import requests
+import httpx
 import socks
 
 from module.conf import settings
@@ -13,40 +13,39 @@
 class RequestURL:
     def __init__(self):
         self.header = {"user-agent": "Mozilla/5.0", "Accept": "application/xml"}
-        self._socks5_proxy = False
 
     def get_url(self, url, retry=3):
         try_time = 0
         while True:
             try:
-                req = self.session.get(url=url, headers=self.header, timeout=5)
-                logger.debug(f"[Network] Successfully connected to {url}. Status: {req.status_code}")
-                req.raise_for_status()
-                return req
-            except requests.RequestException:
+                req = self.session.get(url=url)
                 logger.debug(
-                    f"[Network] Cannot connect to {url}. Wait for 5 seconds."
+                    f"[Network] Successfully connected to {url}. Status: {req.status_code}"
                 )
+                req.raise_for_status()
+                return req
+            except httpx.RequestError:
+                logger.debug(f"[Network] Cannot connect to {url}. Wait for 5 seconds.")
                 try_time += 1
                 if try_time >= retry:
                     break
                 time.sleep(5)
             except Exception as e:
                 logger.debug(e)
                 break
-        logger.error(f"[Network] Unable to connect to {url}, Please check your network settings")
+        logger.error(
+            f"[Network] Unable to connect to {url}, Please check your network settings"
+        )
         return None
 
     def post_url(self, url: str, data: dict, retry=3):
         try_time = 0
         while True:
             try:
-                req = self.session.post(
-                    url=url, headers=self.header, data=data, timeout=5
-                )
+                req = self.session.post(url=url, data=data)
                 req.raise_for_status()
                 return req
-            except requests.RequestException:
+            except httpx.RequestError:
                 logger.warning(
                     f"[Network] Cannot connect to {url}. Wait for 5 seconds."
                 )
@@ -65,60 +64,42 @@ def check_url(self, url: str):
         if "://" not in url:
             url = f"http://{url}"
         try:
-            req = requests.head(url=url, headers=self.header, timeout=5)
+            req = httpx.head(url=url)
             req.raise_for_status()
             return True
-        except requests.RequestException:
+        except httpx.RequestError:
             logger.debug(f"[Network] Cannot connect to {url}.")
             return False
 
     def post_form(self, url: str, data: dict, files):
         try:
-            req = self.session.post(
-                url=url, headers=self.header, data=data, files=files, timeout=5
-            )
+            req = self.session.post(url=url, data=data, files=files)
             req.raise_for_status()
             return req
-        except requests.RequestException:
+        except httpx.RequestError:
             logger.warning(f"[Network] Cannot connect to {url}.")
             return None
 
     def __enter__(self):
-        self.session = requests.Session()
-        if settings.proxy.enable:
-            if "http" in settings.proxy.type:
-                if settings.proxy.username:
-                    username=settings.proxy.username
-                    password=settings.proxy.password
-                    url = f"http://{username}:{password}@{settings.proxy.host}:{settings.proxy.port}"
-                    self.session.proxies = {
-                        "http": url,
-                        "https": url,
-                    }
-                else:
-                    url = f"http://{settings.proxy.host}:{settings.proxy.port}"
-                    self.session.proxies = {
-                        "http": url,
-                        "https": url,
-                    }
-            elif settings.proxy.type == "socks5":
-                self._socks5_proxy = True
-                socks.set_default_proxy(
-                    socks.SOCKS5,
-                    addr=settings.proxy.host,
-                    port=settings.proxy.port,
-                    rdns=True,
-                    username=settings.proxy.username,
-                    password=settings.proxy.password,
-                )
-                socket.socket = socks.socksocket
-            else:
-                logger.error(f"[Network] Unsupported proxy type: {settings.proxy.type}")
+        proxy = self._build_proxy_url() if settings.proxy.enable else None
+        self.session = httpx.Client(
+            headers=self.headers, http2=True, proxies=proxy, timeout=5
+        )
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if self._socks5_proxy:
-            socks.set_default_proxy()
-            socket.socket = socks.socksocket
-            self._socks5_proxy = False
         self.session.close()
+
+    def _build_proxy_url(self):
+        proxy_type = "http" if "http" in settings.proxy.type else "socks5h"
+        auth = (
+            f"{settings.proxy.username}:{settings.proxy.password}@"
+            if settings.proxy.username
+            else ""
+        )
+
+        if proxy_type not in ["http", "socks5h"]:
+            logger.error(f"[Network] Unsupported proxy type: {settings.proxy.type}")
+            return None
+
+        return f"{proxy_type}://{auth}{settings.proxy.host}:{settings.proxy.port}"

backend/src/module/utils/json_config.py

@@ -1,6 +1,6 @@
 import json
 
-import requests
+import httpx
 
 
 def load(filename):
@@ -15,5 +15,5 @@ def save(filename, obj):
 
 
 def get(url):
-    req = requests.get(url)
+    req = httpx.get(url)
     return req.json()