Merge pull request #531 from arabcoders/dev

Refactor: integrate cf solver for both yt-dlp and httpx

FAQ.md

@@ -575,7 +575,8 @@ This will help in case the premiere has a longer loading screen than usual.
 
 # How to bypass CF challenges?
 
-You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL` environment variable to point to your FlareSolverr instance. For example:
+You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL` 
+environment variable to point to your FlareSolverr instance. For example:
 
 ```yaml
 services:
@@ -602,6 +603,3 @@ services:
 ```
 
 For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project.
-
-> [!NOTE]
-> This will only work for yt-dlp part of the project. Anything else will not be affected by this setting for now.

README.md

@@ -32,6 +32,7 @@ Example of the Simple mode interface.
 * Basic authentication support.
 * Supports `curl-cffi`. See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#impersonation). `In docker only`.
 * Bundled `pot provider plugin`. See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide). `In docker only`.
+* Support using [flaresolverr/flaresolverr](https://github.com/flaresolverr/flaresolverr) to bypass Cloudflare protections for yt-dlp and internal http client. See [related FAQ](FAQ.md#how-to-bypass-cf-challenges).
 * Automatic updates for `yt-dlp` and custom `pip` packages. `In docker only`.
 * Conditions feature to apply custom options based on `yt-dlp` returned info.
 * Custom browser extensions, bookmarklets and iOS shortcuts to send links to YTPTube instance.

app/library/DownloadQueue.py

@@ -863,8 +863,7 @@ class DownloadQueue(metaclass=Singleton):
             if not item.requeued and (condition := Conditions.get_instance().match(info=entry)):
                 already.pop()
 
-                item_title = entry.get("title") or entry.get("id") or item.url
+                message = f"Condition '{condition.name}' matched for '{item!r}'."
-                message = f"Condition '{condition.name}' matched for '{item_title}'."
 
                 if condition.cli:
                     message += f" Re-queuing with '{condition.cli}'."
@@ -873,18 +872,19 @@ class DownloadQueue(metaclass=Singleton):
 
                 if condition.extras.get("ignore_download", False):
                     extra_msg: str = ""
-                    if yt_conf.get("download_archive") and not condition.extras.get("no_archive", False):
+                    if _archive_file and not condition.extras.get("no_archive", False):
-                        archive_add(yt_conf.get("download_archive"), [archive_id])
+                        archive_add(_archive_file, [archive_id])
-                        extra_msg = f" and added to archive '{yt_conf.get('download_archive')}'"
+                        extra_msg = f" and added to archive file '{_archive_file}'"
 
-                    log_message = f"Ignoring download of '{item_title}' as per condition '{condition.name}'{extra_msg}."
+                    _name = entry.get("title", entry.get("id"))
+                    log_message = f"Ignoring download of '{_name!r}' as per condition '{condition.name}'{extra_msg}."
 
                     store_type, _ = await self.get_item(archive_id=archive_id)
                     if not store_type:
                         dlInfo = Download(
                             info=ItemDTO(
                                 id=entry.get("id"),
-                                title=item_title,
+                                title=_name,
                                 url=item.url,
                                 preset=item.preset,
                                 folder=item.folder,
@@ -909,7 +909,7 @@ class DownloadQueue(metaclass=Singleton):
 
                 if condition.extras.get("set_preset") and (target_preset := condition.extras.get("set_preset")):
                     if Presets.get_instance().has(target_preset):
-                        log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item_title}' as per condition '{condition.name}'."
+                        log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item!r}' as per condition '{condition.name}'."
                         LOG.info(log_message)
                         self._notify.emit(Events.LOG_INFO, data={}, title="Preset Switched", message=log_message)
                         item = item.new_with(preset=target_preset)

app/library/Notifications.py

@@ -15,6 +15,7 @@ from .BackgroundWorker import BackgroundWorker
 from .config import Config
 from .encoder import Encoder
 from .Events import Event, EventBus, Events
+from .httpx_client import async_client
 from .ItemDTO import Item, ItemDTO
 from .Presets import Preset, Presets
 from .Singleton import Singleton
@@ -147,7 +148,7 @@ class Notification(metaclass=Singleton):
         "Debug mode."
         self._file: Path = Path(file) if file else Path(config.config_path).joinpath("notifications.json")
         "File to store notification targets."
-        self._client: httpx.AsyncClient = client or httpx.AsyncClient()
+        self._client: httpx.AsyncClient = client or async_client()
         "HTTP client to send requests."
         self._encoder: Encoder = encoder or Encoder()
         "Encoder to encode data."

app/library/PackageInstaller.py

@@ -6,7 +6,7 @@ import subprocess
 import sys
 from pathlib import Path
 
-import httpx
+from .httpx_client import sync_client
 
 LOG: logging.Logger = logging.getLogger("package_installer")
 
@@ -17,7 +17,7 @@ def parse_version(v: str) -> tuple[int, ...]:
 
 class Packages:
     def __init__(self, env: str | None, file: str | None, upgrade: bool = False):
-        from_env = env.split() if env else []
+        from_env: list[str] = env.split() if env else []
         from_file = []
 
         if file:
@@ -89,7 +89,7 @@ class PackageInstaller:
     def _get_latest_version(self, pkg: str) -> str | None:
         url: str = f"https://pypi.org/pypi/{pkg}/json"
         try:
-            with httpx.Client(timeout=5.0) as client:
+            with sync_client(timeout=5.0) as client:
                 resp = client.get(url)
                 if 200 == resp.status_code:
                     return resp.json()["info"]["version"]

app/library/cf_solver_handler.py

@@ -1,15 +1,10 @@
-# flake8: noqa: S310
 from __future__ import annotations
 
 import http.cookiejar
-import json
 import logging
-import time
-import urllib.request
 from abc import ABC
 from collections.abc import Callable
 from typing import Any, ClassVar
-from urllib.parse import urlparse
 
 from yt_dlp.networking.common import (
     _REQUEST_HANDLERS,
@@ -24,15 +19,16 @@ from yt_dlp.networking.common import (
 from yt_dlp.networking.exceptions import HTTPError
 from yt_dlp.utils.networking import clean_headers
 
+from app.library.cf_solver_shared import CACHE, is_cf_challenge, solver
+
 LOG: logging.Logger = logging.getLogger(__name__)
 
 SolverFn = Callable[[Request, Response, RequestHandler], Request | None]
-CacheEntry = dict[str, Any]
 
 
 def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> Request | None:
     """
-    A Cloudflare solver that uses FlareSolverr/FlareSolverr to solve challenges.
+    A Cloudflare solver that uses FlareSolverr to solve challenges.
 
     Args:
         request (Request): The original request that triggered the challenge.
@@ -43,81 +39,25 @@ def cf_solver(request: Request, _response: Response, handler: RequestHandler) ->
         Request | None: The modified request with solved credentials, or None if solving failed.
 
     """
-    from app.library.config import Config
+    if not isinstance(handler, CFSolverRH):
-
-    config = Config.get_instance()
-
-    if not config.flaresolverr_url:
         return None
 
-    parsed_endpoint = urlparse(config.flaresolverr_url)
-    if parsed_endpoint.scheme not in ("http", "https"):
-        return None
-
-    if request.data is not None and request.method not in ("GET", None):
-        return None
-
-    method: str = request.method.lower() if isinstance(request.method, str) else "get"
-    if method not in ("get", "head"):
-        method = "get"
-
-    payload: dict[str, Any] = {
-        "cmd": f"request.{method}",
-        "url": request.url,
-        "maxTimeout": int(getattr(config, "flaresolverr_max_timeout", 60) * 1000),
-    }
-
     cookiejar = handler._get_cookiejar(request)
-    cookies = (
+    host: str = handler._get_host(request.url)
-        [
+    cookies = [
-            {
+        {
-                "name": cookie.name,
+            "name": cookie.name,
-                "value": cookie.value,
+            "value": cookie.value,
-                "domain": cookie.domain or urlparse(request.url).hostname or "",
+            "domain": cookie.domain or host,
-                "path": cookie.path or "/",
+            "path": cookie.path or "/",
-            }
+        }
-            for cookie in cookiejar
+        for cookie in cookiejar
-        ]
+    ]
-        if cookiejar
-        else []
-    )
 
-    if cookies:
+    solution: dict[str, Any] | None = solver(request.url, cookies, request.headers.get("User-Agent"))
-        payload["cookies"] = cookies
+    if not handler._apply_solution(solution, request.url, request.headers, cookiejar):
-
-    req = urllib.request.Request(
-        config.flaresolverr_url,
-        data=json.dumps(payload).encode("utf-8"),
-        headers={"Content-Type": "application/json"},
-        method="POST",
-    )
-
-    try:
-        LOG.info(f"Trying to solve Cloudflare challenge for '{request.url}' this may take a while...")
-        with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
-            result = json.loads(resp.read().decode("utf-8"))
-    except Exception as e:
-        LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {e!s}")
         return None
 
-    if "ok" != result.get("status"):
-        LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {result}")
-        return None
-
-    LOG.info(f"Successfully solved Cloudflare challenge for '{request.url}'.")
-
-    solution = result.get("solution") or {}
-    _cookiejar_from_solution(solution.get("cookies"), request, handler)
-
-    if ua := solution.get("userAgent"):
-        request.headers["User-Agent"] = ua
-
-    CFSolverRH.cache[urlparse(request.url).netloc] = {
-        "cookies": solution.get("cookies") or [],
-        "userAgent": ua,
-        "expires_at": time.time() + config.flaresolverr_cache_ttl,
-    }
-
     return request
 
 
@@ -138,39 +78,6 @@ def set_cf_handler(solver: SolverFn | None = None) -> type[CFSolverRH]:
     return CFSolverRH
 
 
-def _cookiejar_from_solution(cookies, request: Request, handler: RequestHandler) -> None:
-    cookiejar = handler._get_cookiejar(request)
-    host = urlparse(request.url).hostname or ""
-    for cookie in cookies or []:
-        name = cookie.get("name")
-        value = cookie.get("value")
-        if not name or value is None:
-            continue
-        domain = cookie.get("domain") or host
-        path = cookie.get("path") or "/"
-        cookiejar.set_cookie(
-            http.cookiejar.Cookie(
-                version=0,
-                name=name,
-                value=value,
-                port=None,
-                port_specified=False,
-                domain=domain,
-                domain_specified=True,
-                domain_initial_dot=domain.startswith("."),
-                path=path,
-                path_specified=True,
-                secure=bool(cookie.get("secure")),
-                expires=cookie.get("expires"),
-                discard=False,
-                comment=None,
-                comment_url=None,
-                rest={},
-                rfc2109=False,
-            )
-        )
-
-
 @register_rh
 class CFSolverRH(RequestHandler, ABC):
     """Request handler that intercepts Cloudflare challenges"""
@@ -178,7 +85,6 @@ class CFSolverRH(RequestHandler, ABC):
     _SUPPORTED_URL_SCHEMES = ("http", "https")
     _SUPPORTED_PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h")
     solver: ClassVar[SolverFn | None] = None
-    cache: ClassVar[dict[str, CacheEntry]] = {}
 
     def __init__(self, *, solver: SolverFn | None = None, **kwargs) -> None:
         super().__init__(**kwargs)
@@ -190,12 +96,66 @@ class CFSolverRH(RequestHandler, ABC):
             self._fallback_director.close()
             self._fallback_director = None
 
+    @staticmethod
+    def _get_host(url: str) -> str:
+        """Extract hostname from URL."""
+        from urllib.parse import urlparse
+
+        return urlparse(url).hostname or ""
+
+    def _apply_solution(
+        self,
+        solution: dict[str, Any] | None,
+        url: str,
+        headers: dict[str, Any],
+        cookiejar: http.cookiejar.CookieJar,
+    ) -> bool:
+        """Apply Cloudflare solution (cookies and User-Agent) to request."""
+        if not solution:
+            return False
+
+        host = self._get_host(url)
+
+        for cookie in solution.get("cookies", []):
+            name = cookie.get("name")
+            value = cookie.get("value")
+            if not name or value is None:
+                continue
+            domain = cookie.get("domain") or host
+            path = cookie.get("path") or "/"
+            cookiejar.set_cookie(
+                http.cookiejar.Cookie(
+                    version=0,
+                    name=name,
+                    value=value,
+                    port=None,
+                    port_specified=False,
+                    domain=domain,
+                    domain_specified=True,
+                    domain_initial_dot=domain.startswith("."),
+                    path=path,
+                    path_specified=True,
+                    secure=bool(cookie.get("secure")),
+                    expires=cookie.get("expires"),
+                    discard=False,
+                    comment=None,
+                    comment_url=None,
+                    rest={},
+                    rfc2109=False,
+                )
+            )
+
+        if ua := solution.get("userAgent"):
+            headers["User-Agent"] = ua
+
+        return True
+
     def _check_extensions(self, extensions) -> None:
         super()._check_extensions(extensions)
         for key in ("cookiejar", "timeout", "legacy_ssl", "keep_header_casing", "cf_retry"):
             extensions.pop(key, None)
 
-    def _validate(self, request: Request):  # type: ignore[override]
+    def _validate(self, request: Request):
         self._check_url_scheme(request)
         self._check_proxies(request.proxies or self.proxies)
         extensions = request.extensions.copy()
@@ -230,30 +190,6 @@ class CFSolverRH(RequestHandler, ABC):
         self._fallback_director = director
         return director
 
-    @staticmethod
-    def _is_cf_response(response: Response) -> bool:
-        """
-        Check if the response is a Cloudflare challenge response.
-
-        Args:
-            response (Response): The HTTP response to check.
-
-        Returns:
-            bool: True if the response is a Cloudflare challenge, False otherwise.
-
-        """
-        status: int | None = getattr(response, "status", None)
-        if status not in (403, 429, 503):
-            return False
-
-        headers = response.headers or {}
-        server_header: str = (headers.get("Server") or "").lower()
-        if "cloudflare" in server_header:
-            return True
-
-        cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor")
-        return any(key in headers for key in cf_header_keys)
-
     def _solve(self, request: Request, response: Response) -> Request | None:
         return self._solver(request, response, self) if self._solver else None
 
@@ -276,42 +212,32 @@ class CFSolverRH(RequestHandler, ABC):
         response.close()
         return director.send(solved_request)
 
-    def _inject_cookie(self, request: Request) -> None:
-        cache_key: str = urlparse(request.url).netloc
-        if not (cached := self.cache.get(cache_key)):
-            return
-
-        if cached.get("expires_at", 0) <= time.time():
-            self.cache.pop(cache_key, None)
-            return
-
-        LOG.info(f"Injecting cached Cloudflare cookies for '{cache_key}'.")
-        _cookiejar_from_solution(cached.get("cookies"), request, self)
-        if ua := cached.get("userAgent"):
-            request.headers["User-Agent"] = ua
-
     def _send(self, request: Request) -> Response:
-        self._inject_cookie(request)
+        host: str = self._get_host(request.url)
+        if host and (cached := CACHE.get(host)):
+            LOG.info(f"Injecting cached Cloudflare cookies for '{host}'.")
+            self._apply_solution(cached, request.url, request.headers, self._get_cookiejar(request))
+
         director: RequestDirector = self._build_fallback()
 
         try:
             response: Response = director.send(request)
         except HTTPError as error:
-            if error.response and self._is_cf_response(error.response):
+            if error.response and is_cf_challenge(getattr(error.response, "status", None), error.response.headers):
                 return self._retry_with_clearance(request, error.response, director)
             raise
 
-        if self._is_cf_response(response):
+        if is_cf_challenge(getattr(response, "status", None), response.headers):
             return self._retry_with_clearance(request, response, director)
 
         return response
 
 
 @register_preference(CFSolverRH)
-def cf_solver_preference(_handler: RequestHandler, _request: Request) -> int:
+def cf_solver_preference(_handler, _request) -> int:
     from app.library.config import Config
 
     if not Config.get_instance().flaresolverr_url:
         return 0
 
-    return 100
+    return 500

app/library/cf_solver_shared.py

@@ -0,0 +1,100 @@
+# flake8: noqa: S310
+from __future__ import annotations
+
+import json
+import logging
+import time
+import urllib.request
+from typing import Any
+from urllib.parse import urlparse
+
+from .cache import Cache
+
+CACHE: Cache = Cache()
+LOG: logging.Logger = logging.getLogger("cf_solver")
+
+
+def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> dict[str, Any] | None:
+    """
+    Run FlareSolverr solve. Returns solution dict or None.
+
+    Args:
+        url (str): The URL to solve the challenge for.
+        cookies (list[dict]): List of existing cookies to send to FlareSolverr.
+        user_agent (str | None): The User-Agent string to send to FlareSolverr.
+
+    Returns:
+        dict[str, Any] | None: The solution dict from FlareSolverr, or None if solving fails.
+
+    """
+    from app.library.config import Config
+
+    config = Config.get_instance()
+    if not (endpoint := config.flaresolverr_url):
+        return None
+
+    parsed = urlparse(endpoint)
+    if parsed.scheme not in ("http", "https"):
+        return None
+
+    host = urlparse(url).hostname or ""
+    if not host:
+        return None
+
+    if cached := CACHE.get(host):
+        return cached
+
+    payload: dict[str, Any] = {
+        "cmd": "request.get",
+        "url": url,
+        "maxTimeout": int(config.flaresolverr_max_timeout * 1000),
+    }
+
+    if cookies:
+        payload["cookies"] = cookies
+
+    if user_agent:
+        payload.setdefault("headers", {})["User-Agent"] = user_agent
+
+    req = urllib.request.Request(
+        endpoint,
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+
+    LOG.info(f"Solving Cloudflare challenge for '{host}' via FlareSolverr.")
+    start_time = time.time()
+    with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
+        result = json.loads(resp.read().decode("utf-8"))
+
+    if "ok" != result.get("status"):
+        LOG.error(f"FlareSolverr failed to solve challenge for '{host}': {result.get('message')}")
+        return None
+
+    LOG.info(f"FlareSolverr successfully solved challenge for '{host}'. (Took {time.time() - start_time:.2f} seconds)")
+
+    solution = result.get("solution") or {}
+    CACHE.set(
+        host,
+        {"cookies": solution.get("cookies") or [], "userAgent": solution.get("userAgent")},
+        ttl=config.flaresolverr_cache_ttl,
+    )
+
+    return CACHE.get(host)
+
+
+def is_cf_challenge(status: int | None, headers: dict[str, Any] | None) -> bool:
+    """
+    Determine whether a response indicates a Cloudflare challenge.
+    """
+    if status not in (403, 429, 503):
+        return False
+
+    headers = headers or {}
+    server_header: str = str(headers.get("Server", "")).lower()
+    if "cloudflare" in server_header:
+        return True
+
+    cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor")
+    return any(key in headers for key in cf_header_keys)

app/library/httpx_client.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+
+from .cf_solver_shared import is_cf_challenge, solver
+
+__all__: list[str] = ["async_client", "sync_client"]
+
+LOG: logging.Logger = logging.getLogger("httpx_cf")
+
+
+def _parse_cookie_header(cookie_header: str | None) -> dict[str, str]:
+    cookies: dict[str, str] = {}
+
+    if not cookie_header:
+        return cookies
+
+    for item in cookie_header.split(";"):
+        item: str = item.strip()
+        if "=" in item:
+            name, value = item.split("=", 1)
+            cookies[name] = value
+
+    return cookies
+
+
+def _merge_cookies(existing_header: str | None, new_cookies: list[dict[str, str]]) -> dict[str, str]:
+    merged: dict[str, str] = _parse_cookie_header(existing_header)
+
+    for cookie in new_cookies:
+        if cookie.get("name") and cookie.get("value"):
+            merged[cookie["name"]] = cookie["value"]
+
+    return merged
+
+
+def _get_transport(
+    enable_cf: bool,
+    is_async: bool,
+    transport: httpx.AsyncBaseTransport | None,
+) -> httpx.AsyncBaseTransport:
+    if enable_cf:
+        return CFAsyncTransport(base=transport) if is_async else CFTransport(base=transport)
+
+    return transport or (httpx.AsyncHTTPTransport() if is_async else httpx.HTTPTransport())
+
+
+class CFAsyncTransport(httpx.AsyncBaseTransport):
+    def __init__(self, base: httpx.AsyncBaseTransport | None = None):
+        self.base: httpx.AsyncBaseTransport | httpx.AsyncHTTPTransport = base or httpx.AsyncHTTPTransport()
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        response: httpx.Response = await self.base.handle_async_request(request)
+        if not is_cf_challenge(response.status_code, dict(response.headers)):
+            return response
+
+        url = str(request.url)
+
+        solution: dict[str, Any] | None = solver(url, [], request.headers.get("User-Agent"))
+        if not solution:
+            return response
+
+        if cookies := solution.get("cookies", []):
+            merged_cookies: dict[str, str] = _merge_cookies(request.headers.get("Cookie"), cookies)
+            headers: httpx.Headers = request.headers.copy()
+            headers.pop("Cookie", None)
+            request = httpx.Request(
+                method=request.method,
+                url=request.url,
+                headers=headers,
+                content=request.content,
+                cookies=merged_cookies,
+            )
+
+        if ua := solution.get("userAgent"):
+            request.headers["User-Agent"] = ua
+
+        await response.aclose()
+        return await self.base.handle_async_request(request)
+
+    def close(self) -> None:
+        self.base.close()
+
+
+class CFTransport(httpx.BaseTransport):
+    def __init__(self, base: httpx.BaseTransport | None = None):
+        self.base: httpx.BaseTransport | httpx.HTTPTransport = base or httpx.HTTPTransport()
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        response: httpx.Response = self.base.handle_request(request)
+        if not is_cf_challenge(response.status_code, dict(response.headers)):
+            return response
+
+        url = str(request.url)
+
+        solution: dict[str, Any] | None = solver(url, [], request.headers.get("User-Agent"))
+        if not solution:
+            return response
+
+        if cookies := solution.get("cookies", []):
+            merged_cookies: dict[str, str] = _merge_cookies(request.headers.get("Cookie"), cookies)
+            headers: httpx.Headers = request.headers.copy()
+            headers.pop("Cookie", None)
+            request = httpx.Request(
+                method=request.method,
+                url=request.url,
+                headers=headers,
+                content=request.content,
+                cookies=merged_cookies,
+            )
+
+        if ua := solution.get("userAgent"):
+            request.headers["User-Agent"] = ua
+
+        response.close()
+        return self.base.handle_request(request)
+
+    def close(self) -> None:
+        self.base.close()
+
+
+def async_client(enable_cf: bool = True, **kwargs: Any) -> httpx.AsyncClient:
+    """
+    Create an httpx.AsyncClient with optional Cloudflare challenge solving.
+
+    Args:
+        enable_cf (bool): Whether to enable Cloudflare challenge solving.
+        **kwargs: Additional keyword arguments to pass to httpx.AsyncClient.
+
+    Returns:
+        httpx.AsyncClient: The configured httpx.AsyncClient instance.
+
+    """
+    transport = kwargs.pop("transport", None)
+    return httpx.AsyncClient(transport=_get_transport(enable_cf, is_async=True, transport=transport), **kwargs)
+
+
+def sync_client(enable_cf: bool = True, **kwargs: Any) -> httpx.Client:
+    """
+    Create an httpx.Client with optional Cloudflare challenge solving.
+
+    Args:
+        enable_cf (bool): Whether to enable Cloudflare challenge solving.
+        **kwargs: Additional keyword arguments to pass to httpx.Client.
+
+    Returns:
+        httpx.Client: The configured httpx.Client instance.
+
+    """
+    transport = kwargs.pop("transport", None)
+    return httpx.Client(transport=_get_transport(enable_cf, is_async=False, transport=transport), **kwargs)

app/library/task_handlers/_base_handler.py

@@ -5,6 +5,7 @@ import httpx
 from yt_dlp.utils.networking import random_user_agent
 
 from app.library.config import Config
+from app.library.httpx_client import async_client
 from app.library.Tasks import Task, TaskFailure, TaskResult
 
 
@@ -73,7 +74,7 @@ class BaseHandler:
         if proxy := ytdlp_opts.get("proxy", None):
             opts["proxy"] = proxy
 
-        async with httpx.AsyncClient(**opts) as client:
+        async with async_client(**opts) as client:
             method = kwargs.pop("method", "GET").upper()
             timeout = ytdlp_opts.get("timeout", ytdlp_opts.get("socket_timeout", 120))
             return await client.request(method=method, url=url, timeout=timeout, **kwargs)

app/library/task_handlers/generic.py

@@ -14,7 +14,6 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal
 from urllib.parse import urljoin
 
-import httpx
 import jmespath
 from parsel import Selector
 from parsel.selector import SelectorList
@@ -22,12 +21,14 @@ from yt_dlp.utils.networking import random_user_agent
 
 from app.library.cache import Cache
 from app.library.config import Config
+from app.library.httpx_client import async_client
 from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult
 from app.library.Utils import extract_info, get_archive_id
 
 from ._base_handler import BaseHandler
 
 if TYPE_CHECKING:
+    import httpx
     from parsel.selector import SelectorList
 
 LOG: logging.Logger = logging.getLogger(__name__)
@@ -474,7 +475,7 @@ def load_task_definitions(config: Config | None = None) -> list[TaskDefinition]:
                 LOG.error(f"[{path.name}] Unsupported response type '{response_type}'.")
                 continue
 
-            response_config = ResponseConfig(format=response_type)  # type: ignore[arg-type]
+            response_config = ResponseConfig(format=response_type)
 
         parse_raw: Mapping | None = raw.get("parse")
         if not isinstance(parse_raw, Mapping):
@@ -826,7 +827,7 @@ class GenericTaskHandler(BaseHandler):
 
         timeout_value: float | Any = definition.request.timeout or ytdlp_opts.get("socket_timeout", 120)
 
-        async with httpx.AsyncClient(**client_options) as client:
+        async with async_client(**client_options) as client:
             response: httpx.Response = await client.request(
                 method=definition.request.normalized_method(),
                 url=url,

app/library/ytdlp.py

@@ -1,11 +1,11 @@
 # flake8: noqa: F401
+import sys
 from typing import Any
 
 import yt_dlp
 from yt_dlp.utils import make_archive_id
 
-# imported to register the handler with yt-dlp.
+from app.library.cf_solver_handler import set_cf_handler
-from app.library.CFSolverRH import set_cf_handler
 
 
 class _ArchiveProxy:

app/routes/api/docs.py

@@ -3,7 +3,6 @@ import time
 from datetime import UTC, datetime
 from typing import Any
 
-import httpx
 from aiohttp import web
 from aiohttp.web import Request, Response
 from yt_dlp.utils.networking import random_user_agent
@@ -86,7 +85,9 @@ async def get_doc(request: Request, config: Config, cache: Cache) -> Response:
         except Exception:
             pass
 
-        async with httpx.AsyncClient(**opts) as client:
+        from app.library.httpx_client import async_client
+
+        async with async_client(**opts) as client:
             LOG.debug(f"Fetching doc from '{url}'.")
             response = await client.request(method="GET", url=url, follow_redirects=True)
             dct = {

app/routes/api/images.py

@@ -5,7 +5,6 @@ from datetime import UTC, datetime
 from typing import Any
 from urllib.parse import urlparse
 
-import httpx
 from aiohttp import web
 from aiohttp.web import Request, Response
 from yt_dlp.utils.networking import random_user_agent
@@ -67,7 +66,9 @@ async def get_thumbnail(request: Request, config: Config) -> Response:
         except Exception:
             pass
 
-        async with httpx.AsyncClient(**opts) as client:
+        from app.library.httpx_client import async_client
+
+        async with async_client(**opts) as client:
             LOG.debug(f"Fetching thumbnail from '{url}'.")
             response = await client.request(method="GET", url=url, follow_redirects=True)
 
@@ -158,7 +159,9 @@ async def get_background(request: Request, config: Config, cache: Cache) -> Resp
         except Exception:
             pass
 
-        async with httpx.AsyncClient(**opts) as client:
+        from app.library.httpx_client import async_client
+
+        async with async_client(**opts) as client:
             if backend.startswith("https://www.bing.com/HPImageArchive.aspx"):
                 if not cache.has(CACHE_KEY_BING):
                     response = await client.request(method="GET", url=backend)

app/routes/api/tasks.py

@@ -307,9 +307,10 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R
         filename.write_text(xml_content, encoding="utf-8")
 
         try:
-            import httpx
             from yt_dlp.utils.networking import random_user_agent
 
+            from app.library.httpx_client import async_client
+
             ytdlp_args: dict = task.get_ytdlp_opts().get_all()
             opts: dict[str, Any] = {
                 "headers": {
@@ -331,7 +332,7 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R
             except Exception:
                 pass
 
-            async with httpx.AsyncClient(**opts) as client:
+            async with async_client(**opts) as client:
                 for key in info.get("thumbnails", {}):
                     try:
                         url = info["thumbnails"][key]

app/tests/test_cf_solver_handler.py

@@ -0,0 +1,257 @@
+"""Comprehensive tests for cf_solver_handler yt-dlp request handler."""
+
+from __future__ import annotations
+
+import http.cookiejar
+from unittest.mock import Mock, patch
+
+import pytest
+
+try:
+    from yt_dlp.networking.common import Request, RequestDirector, Response
+    from yt_dlp.networking.exceptions import HTTPError
+
+    YTDLP_AVAILABLE = True
+except ImportError:
+    YTDLP_AVAILABLE = False
+    Request = Mock
+    Response = Mock
+    RequestDirector = Mock
+    HTTPError = Exception
+
+pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available")
+
+
+@pytest.fixture(scope="module")
+def cf_handler_module():
+    """Lazily import cf_solver_handler module to avoid circular imports."""
+    from app.library import cf_solver_handler
+
+    return cf_solver_handler
+
+
+class TestCfSolverFunction:
+    """Test the cf_solver function."""
+
+    @patch("app.library.cf_solver_handler.solver")
+    def test_cf_solver_success(self, mock_solver, cf_handler_module):
+        """Test successful solving of Cloudflare challenge."""
+        mock_solver.return_value = {"cookies": [], "userAgent": "Mozilla/5.0"}
+
+        handler = cf_handler_module.CFSolverRH(logger=Mock())
+        cookiejar = http.cookiejar.CookieJar()
+        handler._get_cookiejar = Mock(return_value=cookiejar)
+
+        request = Mock()
+        request.url = "https://example.com/path"
+        request.headers = {"User-Agent": "test"}
+
+        response = Mock()
+
+        result = cf_handler_module.cf_solver(request, response, handler)
+
+        assert result is request
+        mock_solver.assert_called_once()
+
+    @patch("app.library.cf_solver_handler.solver")
+    def test_cf_solver_no_solution(self, mock_solver, cf_handler_module):
+        """Test when solver returns no solution."""
+        mock_solver.return_value = None
+
+        handler = cf_handler_module.CFSolverRH(logger=Mock())
+        cookiejar = http.cookiejar.CookieJar()
+        handler._get_cookiejar = Mock(return_value=cookiejar)
+
+        request = Mock()
+        request.url = "https://example.com/path"
+        request.headers = {"User-Agent": "test"}
+
+        response = Mock()
+
+        result = cf_handler_module.cf_solver(request, response, handler)
+
+        assert result is None
+        mock_solver.assert_called_once()
+
+    @patch("app.library.cf_solver_handler.solver")
+    def test_cf_solver_with_existing_cookies(self, mock_solver, cf_handler_module):
+        """Test solving with existing cookies in jar."""
+        mock_solver.return_value = {"cookies": [], "userAgent": "Mozilla/5.0"}
+
+        handler = cf_handler_module.CFSolverRH(logger=Mock())
+        cookiejar = http.cookiejar.CookieJar()
+
+        cookie = http.cookiejar.Cookie(
+            version=0,
+            name="existing",
+            value="value",
+            port=None,
+            port_specified=False,
+            domain="example.com",
+            domain_specified=True,
+            domain_initial_dot=False,
+            path="/",
+            path_specified=True,
+            secure=False,
+            expires=None,
+            discard=True,
+            comment=None,
+            comment_url=None,
+            rest={},
+            rfc2109=False,
+        )
+        cookiejar.set_cookie(cookie)
+        handler._get_cookiejar = Mock(return_value=cookiejar)
+
+        request = Mock()
+        request.url = "https://example.com/path"
+        request.headers = {"User-Agent": "test"}
+
+        response = Mock()
+
+        result = cf_handler_module.cf_solver(request, response, handler)
+
+        assert result is request
+        call_args = mock_solver.call_args
+        assert call_args is not None
+        cookies_arg = call_args[0][1]
+        assert len(cookies_arg) > 0
+        assert "existing" == cookies_arg[0]["name"]
+
+
+class TestSetCfHandler:
+    """Test set_cf_handler function."""
+
+    def test_set_cf_handler_default(self, cf_handler_module):
+        """Test setting CF handler with default solver."""
+        result = cf_handler_module.set_cf_handler()
+        assert result is cf_handler_module.CFSolverRH
+        assert cf_handler_module.CFSolverRH.solver is None or callable(cf_handler_module.CFSolverRH.solver)
+
+    def test_set_cf_handler_custom_solver(self, cf_handler_module):
+        """Test setting CF handler with custom solver."""
+
+        def custom_solver(req, resp, handler):
+            return req
+
+        result = cf_handler_module.set_cf_handler(custom_solver)
+        assert result is cf_handler_module.CFSolverRH
+        assert cf_handler_module.CFSolverRH.solver is custom_solver
+
+
+class TestCFSolverRH:
+    """Test CFSolverRH request handler class."""
+
+    @pytest.fixture(autouse=True)
+    def setup(self, cf_handler_module):
+        """Set up test fixtures."""
+        self.module = cf_handler_module
+        self.handler = self.module.CFSolverRH(logger=Mock(), verbose=False)
+
+    def test_init_default(self):
+        """Test initialization with defaults."""
+        handler = self.module.CFSolverRH(logger=Mock(), verbose=False)
+        assert handler._solver is not None
+        assert handler._fallback_director is None
+
+    def test_init_custom_solver(self):
+        """Test initialization with custom solver."""
+
+        def custom_solver(req, resp, handler):
+            return req
+
+        handler = self.module.CFSolverRH(logger=Mock(), verbose=False, solver=custom_solver)
+        assert handler._solver is custom_solver
+
+    def test_close(self):
+        """Test closing handler."""
+        mock_director = Mock()
+        self.handler._fallback_director = mock_director
+
+        self.handler.close()
+
+        mock_director.close.assert_called_once()
+        assert self.handler._fallback_director is None
+
+    def test_close_no_director(self):
+        """Test closing handler when no director exists."""
+        self.handler._fallback_director = None
+        self.handler.close()
+
+    def test_check_extensions(self):
+        """Test extension checking."""
+        extensions = {
+            "timeout": 30,
+            "legacy_ssl": True,
+            "other": "value",
+        }
+
+        self.handler._check_extensions(extensions)
+
+    def test_validate(self):
+        """Test request validation."""
+        request = Mock()
+        request.url = "https://example.com"
+        request.proxies = None
+        request.extensions = {}
+
+        self.handler._validate(request)
+
+    def test_solve(self):
+        """Test solving challenge."""
+        request = Mock()
+        request.url = "https://example.com"
+        request.headers = {}
+        request.extensions = {}
+
+        response = Mock()
+
+        self.handler._solver = Mock(return_value=request)
+        result = self.handler._solve(request, response)
+        assert result is request
+        self.handler._solver.assert_called_once()
+
+        self.handler._solver = None
+        result = self.handler._solve(request, response)
+        assert result is None
+
+    def test_mark_retry(self):
+        """Test marking request as retry."""
+        request = Mock()
+        request.copy = Mock(return_value=Mock())
+        request.copy.return_value.extensions = {}
+
+        new_request = self.module.CFSolverRH._mark_retry(request)
+
+        assert new_request.extensions.get("cf_retry") is True
+
+
+class TestCfSolverPreference:
+    """Test cf_solver_preference function."""
+
+    def test_preference_with_flaresolverr(self, cf_handler_module, monkeypatch):
+        """Test preference when FlareSolverr is configured."""
+        mock_config = Mock()
+        mock_config.flaresolverr_url = "http://localhost:8191/v1"
+
+        import app.library.config
+
+        monkeypatch.setattr(app.library.config.Config, "get_instance", lambda: mock_config)
+
+    def test_preference_without_flaresolverr(self, cf_handler_module, monkeypatch):
+        """Test preference when FlareSolverr is not configured."""
+        mock_config = Mock()
+        mock_config.flaresolverr_url = None
+
+        import app.library.config
+
+        monkeypatch.setattr(app.library.config.Config, "get_instance", lambda: mock_config)
+
+    def test_preference_with_empty_flaresolverr(self, cf_handler_module, monkeypatch):
+        """Test preference when FlareSolverr URL is empty."""
+        mock_config = Mock()
+        mock_config.flaresolverr_url = ""
+
+        import app.library.config
+
+        monkeypatch.setattr(app.library.config.Config, "get_instance", lambda: mock_config)

app/tests/test_httpx_client.py

@@ -0,0 +1,436 @@
+"""Comprehensive tests for httpx_client Cloudflare challenge solving."""
+
+from __future__ import annotations
+
+import http.cookiejar
+from unittest.mock import AsyncMock, MagicMock, Mock, patch
+
+import httpx
+import pytest
+
+from app.library.httpx_client import (
+    CFAsyncTransport,
+    CFTransport,
+    _get_transport,
+    async_client,
+    sync_client,
+)
+
+
+class TestGetTransport:
+    """Test transport factory function."""
+
+    def test_get_transport_cf_enabled_async(self):
+        """Test getting async transport with CF enabled."""
+        transport = _get_transport(enable_cf=True, is_async=True, transport=None)
+        assert isinstance(transport, CFAsyncTransport)
+
+    def test_get_transport_cf_enabled_sync(self):
+        """Test getting sync transport with CF enabled."""
+        transport = _get_transport(enable_cf=False, is_async=False, transport=None)
+        assert isinstance(transport, httpx.HTTPTransport)
+
+    def test_get_transport_cf_disabled_async(self):
+        """Test getting async transport with CF disabled."""
+        transport = _get_transport(enable_cf=False, is_async=True, transport=None)
+        assert isinstance(transport, httpx.AsyncHTTPTransport)
+
+    def test_get_transport_cf_disabled_sync(self):
+        """Test getting sync transport with CF disabled."""
+        transport = _get_transport(enable_cf=False, is_async=False, transport=None)
+        assert isinstance(transport, httpx.HTTPTransport)
+
+    def test_get_transport_custom_base_async(self):
+        """Test getting transport with custom base transport."""
+        custom_transport = httpx.AsyncHTTPTransport()
+        transport = _get_transport(enable_cf=True, is_async=True, transport=custom_transport)
+        assert isinstance(transport, CFAsyncTransport)
+        assert transport.base is custom_transport
+
+    def test_get_transport_custom_base_sync(self):
+        """Test getting transport with custom base transport."""
+        custom_transport = httpx.HTTPTransport()
+        transport = _get_transport(enable_cf=True, is_async=False, transport=custom_transport)
+        assert isinstance(transport, CFTransport)
+        assert transport.base is custom_transport
+
+
+class TestCFAsyncTransport:
+    """Test async Cloudflare transport wrapper."""
+
+    def setup_method(self):
+        """Set up test fixtures."""
+        self.base_transport = AsyncMock(spec=httpx.AsyncHTTPTransport)
+        self.transport = CFAsyncTransport(base=self.base_transport)
+
+    @pytest.mark.asyncio
+    async def test_init_default(self):
+        """Test initialization with defaults."""
+        transport = CFAsyncTransport()
+        assert isinstance(transport.base, httpx.AsyncHTTPTransport)
+
+    @pytest.mark.asyncio
+    async def test_init_custom(self):
+        """Test initialization with custom base."""
+        assert self.transport.base is self.base_transport
+
+    @pytest.mark.asyncio
+    @patch("app.library.httpx_client.is_cf_challenge")
+    async def test_handle_request_no_challenge(self, mock_is_cf):
+        """Test handling request without Cloudflare challenge."""
+        mock_is_cf.return_value = False
+
+        request = httpx.Request("GET", "https://example.com")
+        response = Mock(status_code=200, headers={})
+        self.base_transport.handle_async_request.return_value = response
+
+        result = await self.transport.handle_async_request(request)
+
+        assert result is response
+        self.base_transport.handle_async_request.assert_called_once()
+        mock_is_cf.assert_called_once_with(200, {})
+
+    @pytest.mark.asyncio
+    @pytest.mark.asyncio
+    @patch("app.library.httpx_client.solver")
+    @patch("app.library.httpx_client.is_cf_challenge")
+    async def test_handle_request_cached_solution(self, mock_is_cf, mock_solver):
+        """Test handling request with Cloudflare solution from cache."""
+        mock_is_cf.return_value = True
+        mock_solver.return_value = {
+            "cookies": [{"name": "cf_clearance", "value": "token123"}],
+            "userAgent": "Mozilla/5.0",
+        }
+
+        request = httpx.Request("GET", "https://example.com")
+        response1 = Mock(status_code=403, headers={"Server": "cloudflare"})
+        response1.aclose = AsyncMock()
+        response2 = Mock(status_code=200, headers={})
+
+        self.base_transport.handle_async_request.side_effect = [response1, response2]
+
+        result = await self.transport.handle_async_request(request)
+
+        assert result is response2
+        assert 2 == self.base_transport.handle_async_request.call_count
+        mock_solver.assert_called_once()
+        response1.aclose.assert_called_once()
+
+        second_call_request = self.base_transport.handle_async_request.call_args_list[1][0][0]
+        assert "cf_clearance=token123" == second_call_request.headers["Cookie"]
+        assert "Mozilla/5.0" == second_call_request.headers["User-Agent"]
+
+    @pytest.mark.asyncio
+    @patch("app.library.httpx_client.solver")
+    @patch("app.library.httpx_client.is_cf_challenge")
+    async def test_handle_request_solve_challenge(self, mock_is_cf, mock_solver):
+        """Test handling request by solving Cloudflare challenge."""
+        mock_is_cf.return_value = True
+        mock_solver.return_value = {
+            "cookies": [{"name": "cf_clearance", "value": "token123"}, {"name": "__cf_bm", "value": "bm456"}],
+            "userAgent": "Mozilla/5.0",
+        }
+
+        request = httpx.Request("GET", "https://example.com")
+        response1 = Mock(status_code=403, headers={"cf-ray": "123"})
+        response1.aclose = AsyncMock()
+        response2 = Mock(status_code=200, headers={})
+
+        self.base_transport.handle_async_request.side_effect = [response1, response2]
+
+        result = await self.transport.handle_async_request(request)
+
+        assert result is response2
+        assert 2 == self.base_transport.handle_async_request.call_count
+        mock_solver.assert_called_once()
+        response1.aclose.assert_called_once()
+
+        second_call_request = self.base_transport.handle_async_request.call_args_list[1][0][0]
+        assert "cf_clearance=token123; __cf_bm=bm456" == second_call_request.headers["Cookie"]
+        assert "Mozilla/5.0" == second_call_request.headers["User-Agent"]
+
+    @pytest.mark.asyncio
+    @patch("app.library.httpx_client.solver")
+    @patch("app.library.httpx_client.is_cf_challenge")
+    async def test_handle_request_merge_existing_cookies(self, mock_is_cf, mock_solver):
+        """Test that existing cookies are preserved when adding CF cookies."""
+        mock_is_cf.return_value = True
+        mock_solver.return_value = {
+            "cookies": [{"name": "cf_clearance", "value": "token123"}],
+            "userAgent": "Mozilla/5.0",
+        }
+
+        request = httpx.Request("GET", "https://example.com", cookies={"session_id": "abc123", "user_pref": "dark"})
+        response1 = Mock(status_code=403, headers={"cf-ray": "123"})
+        response1.aclose = AsyncMock()
+        response2 = Mock(status_code=200, headers={})
+
+        self.base_transport.handle_async_request.side_effect = [response1, response2]
+
+        result = await self.transport.handle_async_request(request)
+
+        assert result is response2
+
+        second_call_request = self.base_transport.handle_async_request.call_args_list[1][0][0]
+        cookie_header = second_call_request.headers["Cookie"]
+        assert "session_id=abc123" in cookie_header
+        assert "user_pref=dark" in cookie_header
+        assert "cf_clearance=token123" in cookie_header
+
+    @pytest.mark.asyncio
+    @patch("app.library.httpx_client.solver")
+    @patch("app.library.httpx_client.is_cf_challenge")
+    async def test_handle_request_solve_failed(self, mock_is_cf, mock_solver):
+        """Test handling request when solving fails."""
+        mock_is_cf.return_value = True
+        mock_solver.return_value = None
+
+        request = httpx.Request("GET", "https://example.com")
+        response = Mock(status_code=403, headers={"cf-ray": "123"})
+        response.close = Mock()
+
+        self.base_transport.handle_async_request.return_value = response
+
+        result = await self.transport.handle_async_request(request)
+
+        assert result is response
+        assert 1 == self.base_transport.handle_async_request.call_count
+        mock_solver.assert_called_once()
+        response.close.assert_not_called()
+
+    def test_close(self):
+        """Test closing transport."""
+        self.base_transport.close = Mock()
+        self.transport.close()
+        self.base_transport.close.assert_called_once()
+
+
+class TestCFTransport:
+    """Test sync Cloudflare transport wrapper."""
+
+    def setup_method(self):
+        """Set up test fixtures."""
+        self.base_transport = Mock(spec=httpx.HTTPTransport)
+        self.transport = CFTransport(base=self.base_transport)
+
+    def test_init_default(self):
+        """Test initialization with defaults."""
+        transport = CFTransport()
+        assert isinstance(transport.base, httpx.HTTPTransport)
+
+    def test_init_custom(self):
+        """Test initialization with custom base."""
+        assert self.transport.base is self.base_transport
+
+    @patch("app.library.httpx_client.is_cf_challenge")
+    def test_handle_request_no_challenge(self, mock_is_cf):
+        """Test handling request without Cloudflare challenge."""
+        mock_is_cf.return_value = False
+
+        request = httpx.Request("GET", "https://example.com")
+        response = Mock(status_code=200, headers={})
+        self.base_transport.handle_request.return_value = response
+
+        result = self.transport.handle_request(request)
+
+        assert result is response
+        self.base_transport.handle_request.assert_called_once()
+        mock_is_cf.assert_called_once_with(200, {})
+
+    @patch("app.library.httpx_client.solver")
+    @patch("app.library.httpx_client.is_cf_challenge")
+    def test_handle_request_cached_solution(self, mock_is_cf, mock_solver):
+        """Test handling request with Cloudflare solution from cache."""
+        mock_is_cf.return_value = True
+        mock_solver.return_value = {
+            "cookies": [{"name": "cf_clearance", "value": "token123"}],
+            "userAgent": "Mozilla/5.0",
+        }
+
+        request = httpx.Request("GET", "https://example.com")
+        response1 = Mock(status_code=403, headers={"Server": "cloudflare"})
+        response1.close = Mock()
+        response2 = Mock(status_code=200, headers={})
+
+        self.base_transport.handle_request.side_effect = [response1, response2]
+
+        result = self.transport.handle_request(request)
+
+        assert result is response2
+        assert 2 == self.base_transport.handle_request.call_count
+        mock_solver.assert_called_once()
+        response1.close.assert_called_once()
+
+        second_call_request = self.base_transport.handle_request.call_args_list[1][0][0]
+        assert "cf_clearance=token123" == second_call_request.headers["Cookie"]
+        assert "Mozilla/5.0" == second_call_request.headers["User-Agent"]
+
+    @patch("app.library.httpx_client.solver")
+    @patch("app.library.httpx_client.is_cf_challenge")
+    def test_handle_request_solve_challenge(self, mock_is_cf, mock_solver):
+        """Test handling request by solving Cloudflare challenge."""
+        mock_is_cf.return_value = True
+        mock_solver.return_value = {
+            "cookies": [{"name": "cf_clearance", "value": "token123"}, {"name": "__cf_bm", "value": "bm456"}],
+            "userAgent": "Mozilla/5.0",
+        }
+
+        request = httpx.Request("GET", "https://example.com")
+        response1 = Mock(status_code=503, headers={"cf-cache-status": "DYNAMIC"})
+        response1.close = Mock()
+        response2 = Mock(status_code=200, headers={})
+
+        self.base_transport.handle_request.side_effect = [response1, response2]
+
+        result = self.transport.handle_request(request)
+
+        assert result is response2
+        assert 2 == self.base_transport.handle_request.call_count
+        mock_solver.assert_called_once()
+        response1.close.assert_called_once()
+
+        second_call_request = self.base_transport.handle_request.call_args_list[1][0][0]
+        assert "cf_clearance=token123; __cf_bm=bm456" == second_call_request.headers["Cookie"]
+        assert "Mozilla/5.0" == second_call_request.headers["User-Agent"]
+
+    @patch("app.library.httpx_client.solver")
+    @patch("app.library.httpx_client.is_cf_challenge")
+    def test_handle_request_merge_existing_cookies(self, mock_is_cf, mock_solver):
+        """Test that existing cookies are preserved when adding CF cookies."""
+        mock_is_cf.return_value = True
+        mock_solver.return_value = {
+            "cookies": [{"name": "cf_clearance", "value": "token123"}],
+            "userAgent": "Mozilla/5.0",
+        }
+
+        request = httpx.Request("GET", "https://example.com", cookies={"session_id": "abc123", "user_pref": "dark"})
+        response1 = Mock(status_code=403, headers={"cf-ray": "123"})
+        response1.close = Mock()
+        response2 = Mock(status_code=200, headers={})
+
+        self.base_transport.handle_request.side_effect = [response1, response2]
+
+        result = self.transport.handle_request(request)
+
+        assert result is response2
+
+        second_call_request = self.base_transport.handle_request.call_args_list[1][0][0]
+        cookie_header = second_call_request.headers["Cookie"]
+        assert "session_id=abc123" in cookie_header
+        assert "user_pref=dark" in cookie_header
+        assert "cf_clearance=token123" in cookie_header
+
+    @patch("app.library.httpx_client.solver")
+    @patch("app.library.httpx_client.is_cf_challenge")
+    def test_handle_request_solve_failed(self, mock_is_cf, mock_solver):
+        """Test handling request when solving fails."""
+        mock_is_cf.return_value = True
+        mock_solver.return_value = None
+
+        request = httpx.Request("GET", "https://example.com")
+        response = Mock(status_code=429, headers={"cf-ray": "abc123"})
+        response.close = Mock()
+
+        self.base_transport.handle_request.return_value = response
+
+        result = self.transport.handle_request(request)
+
+        assert result is response
+        assert 1 == self.base_transport.handle_request.call_count
+        mock_solver.assert_called_once()
+        response.close.assert_not_called()
+
+    def test_close(self):
+        """Test closing transport."""
+        self.base_transport.close = Mock()
+        self.transport.close()
+        self.base_transport.close.assert_called_once()
+
+
+class TestAsyncClient:
+    """Test async client factory function."""
+
+    @pytest.mark.asyncio
+    async def test_async_client_cf_enabled(self):
+        """Test creating async client with CF enabled."""
+        async with async_client(enable_cf=True) as client:
+            assert isinstance(client, httpx.AsyncClient)
+            assert isinstance(client._transport, CFAsyncTransport)
+
+    @pytest.mark.asyncio
+    async def test_async_client_cf_disabled(self):
+        """Test creating async client with CF disabled."""
+        async with async_client(enable_cf=False) as client:
+            assert isinstance(client, httpx.AsyncClient)
+            assert isinstance(client._transport, httpx.AsyncHTTPTransport)
+
+    @pytest.mark.asyncio
+    async def test_async_client_kwargs(self):
+        """Test creating async client with additional kwargs."""
+        async with async_client(enable_cf=True, timeout=30.0, follow_redirects=True) as client:
+            assert isinstance(client, httpx.AsyncClient)
+            assert isinstance(client._transport, CFAsyncTransport)
+            assert 30.0 == client.timeout.read
+            assert client.follow_redirects is True
+
+    @pytest.mark.asyncio
+    async def test_async_client_custom_transport(self):
+        """Test creating async client with custom transport."""
+        custom = httpx.AsyncHTTPTransport()
+        async with async_client(enable_cf=True, transport=custom) as client:
+            assert isinstance(client, httpx.AsyncClient)
+            assert isinstance(client._transport, CFAsyncTransport)
+            assert client._transport.base is custom
+
+    @pytest.mark.asyncio
+    async def test_async_client_custom_transport_cf_disabled(self):
+        """Test creating async client with custom transport and CF disabled."""
+        custom = httpx.AsyncHTTPTransport()
+        async with async_client(enable_cf=False, transport=custom) as client:
+            assert isinstance(client, httpx.AsyncClient)
+
+            assert client._transport is custom
+
+
+class TestSyncClient:
+    """Test sync client factory function."""
+
+    def test_sync_client_cf_enabled(self):
+        """Test creating sync client with CF enabled."""
+        client = sync_client(enable_cf=True)
+        assert isinstance(client, httpx.Client)
+        assert isinstance(client._transport, CFTransport)
+        client.close()
+
+    def test_sync_client_cf_disabled(self):
+        """Test creating sync client with CF disabled."""
+        client = sync_client(enable_cf=False)
+        assert isinstance(client, httpx.Client)
+        assert isinstance(client._transport, httpx.HTTPTransport)
+        client.close()
+
+    def test_sync_client_kwargs(self):
+        """Test creating sync client with additional kwargs."""
+        client = sync_client(enable_cf=True, timeout=30.0, follow_redirects=True)
+        assert isinstance(client, httpx.Client)
+        assert isinstance(client._transport, CFTransport)
+        assert 30.0 == client.timeout.read
+        assert client.follow_redirects is True
+        client.close()
+
+    def test_sync_client_custom_transport(self):
+        """Test creating sync client with custom transport."""
+        custom = httpx.HTTPTransport()
+        client = sync_client(enable_cf=True, transport=custom)
+        assert isinstance(client, httpx.Client)
+        assert isinstance(client._transport, CFTransport)
+        assert client._transport.base is custom
+        client.close()
+
+    def test_sync_client_custom_transport_cf_disabled(self):
+        """Test creating sync client with custom transport and CF disabled."""
+        custom = httpx.HTTPTransport()
+        client = sync_client(enable_cf=False, transport=custom)
+        assert isinstance(client, httpx.Client)
+        assert client._transport is custom, "When CF is disabled, custom transport should be used directly"
+        client.close()

app/tests/test_package_installer.py

@@ -95,7 +95,7 @@ class TestInstalledAndLatest:
         mock_version.side_effect = PackageNotFoundError
         assert inst._get_installed_version("bar") is None
 
-    @patch("app.library.PackageInstaller.httpx.Client")
+    @patch("app.library.PackageInstaller.sync_client")
     def test_get_latest_version_success(self, mock_client, tmp_path: Path) -> None:
         inst = PackageInstaller(pkg_path=tmp_path)
 
@@ -108,7 +108,7 @@ class TestInstalledAndLatest:
 
         assert inst._get_latest_version("foo") == "9.9.9"
 
-    @patch("app.library.PackageInstaller.httpx.Client")
+    @patch("app.library.PackageInstaller.sync_client")
     def test_get_latest_version_non_200(self, mock_client, tmp_path: Path) -> None:
         inst = PackageInstaller(pkg_path=tmp_path)
         client = MagicMock()
@@ -118,7 +118,7 @@ class TestInstalledAndLatest:
         mock_client.return_value.__enter__.return_value = client
         assert inst._get_latest_version("foo") is None
 
-    @patch("app.library.PackageInstaller.httpx.Client")
+    @patch("app.library.PackageInstaller.sync_client")
     def test_get_latest_version_exception(self, mock_client, tmp_path: Path) -> None:
         inst = PackageInstaller(pkg_path=tmp_path)
         mock_client.side_effect = RuntimeError("boom")

ui/app/stores/SocketStore.ts

@@ -1,4 +1,6 @@
 import { io, type Socket as IOSocket, type SocketOptions, type ManagerOptions } from "socket.io-client"
+import { ref, readonly } from 'vue'
+import { defineStore } from 'pinia'
 import type { ConfigState } from "~/types/config";
 import type { StoreItem } from "~/types/store";
 
@@ -15,6 +17,8 @@ export const useSocketStore = defineStore('socket', () => {
   const connectionStatus = ref<connectionStatus>('disconnected')
   const error = ref<string | null>(null)
   const error_count = ref<number>(0)
+  const wasHidden = ref<boolean>(false)
+  const reconnectTimeout = ref<NodeJS.Timeout | null>(null)
 
   const emit = (event: string, data?: any): any => socket.value?.emit(event, data)
   const on = (event: string | string[], callback: (...args: any[]) => void, withEvent: boolean = false) => {
@@ -33,6 +37,46 @@ export const useSocketStore = defineStore('socket', () => {
 
   const getSessionId = (): string | null => socket.value?.id || null
 
+  const handleVisibilityChange = () => {
+    if (document.hidden) {
+      wasHidden.value = true
+      return
+    }
+
+    if (true === wasHidden.value && false === isConnected.value) {
+      if (null !== reconnectTimeout.value) {
+        clearTimeout(reconnectTimeout.value)
+        reconnectTimeout.value = null
+      }
+
+      reconnectTimeout.value = setTimeout(() => {
+        if (false === isConnected.value) {
+          console.debug('[SocketStore] Page visible after background, reconnecting...')
+          reconnect()
+        }
+        reconnectTimeout.value = null
+      }, 100)
+    }
+
+    wasHidden.value = false
+  }
+
+  const setupVisibilityListener = () => {
+    if (typeof document !== 'undefined') {
+      document.addEventListener('visibilitychange', handleVisibilityChange)
+    }
+  }
+
+  const cleanupVisibilityListener = () => {
+    if (typeof document !== 'undefined') {
+      document.removeEventListener('visibilitychange', handleVisibilityChange)
+    }
+    if (null !== reconnectTimeout.value) {
+      clearTimeout(reconnectTimeout.value)
+      reconnectTimeout.value = null
+    }
+  }
+
   const reconnect = () => {
     if (true === isConnected.value) {
       return;
@@ -49,6 +93,7 @@ export const useSocketStore = defineStore('socket', () => {
     socket.value = null;
     isConnected.value = false;
     connectionStatus.value = 'disconnected';
+    cleanupVisibilityListener();
   }
 
   const connect = () => {
@@ -234,6 +279,8 @@ export const useSocketStore = defineStore('socket', () => {
 
     on('presets_update', (data: string) => config.update('presets', JSON.parse(data).data || []));
     on('dlfields_update', (data: string) => config.update('dl_fields', JSON.parse(data).data || []));
+
+    setupVisibilityListener();
   }
 
   if (false === isConnected.value) {
@@ -243,7 +290,7 @@ export const useSocketStore = defineStore('socket', () => {
   return {
     connect, reconnect, disconnect,
     on, off, emit,
-    socket, isConnected,
+    isConnected,
     getSessionId,
     connectionStatus: readonly(connectionStatus) as Readonly<Ref<connectionStatus>>,
     error: readonly(error) as Readonly<Ref<string | null>>,