diff --git a/FAQ.md b/FAQ.md index e9420737..e8d23c54 100644 --- a/FAQ.md +++ b/FAQ.md @@ -575,7 +575,8 @@ This will help in case the premiere has a longer loading screen than usual. # How to bypass CF challenges? -You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL` environment variable to point to your FlareSolverr instance. For example: +You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL` +environment variable to point to your FlareSolverr instance. For example: ```yaml services: @@ -602,6 +603,3 @@ services: ``` For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project. - -> [!NOTE] -> This will only work for yt-dlp part of the project. Anything else will not be affected by this setting for now. diff --git a/README.md b/README.md index e0a51afc..36079ef7 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Example of the Simple mode interface. * Basic authentication support. * Supports `curl-cffi`. See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#impersonation). `In docker only`. * Bundled `pot provider plugin`. See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide). `In docker only`. +* Support using [flaresolverr/flaresolverr](https://github.com/flaresolverr/flaresolverr) to bypass Cloudflare protections for yt-dlp and internal http client. See [related FAQ](FAQ.md#how-to-bypass-cf-challenges). * Automatic updates for `yt-dlp` and custom `pip` packages. `In docker only`. * Conditions feature to apply custom options based on `yt-dlp` returned info. * Custom browser extensions, bookmarklets and iOS shortcuts to send links to YTPTube instance. diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 11599fd4..795a201e 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -863,8 +863,7 @@ class DownloadQueue(metaclass=Singleton): if not item.requeued and (condition := Conditions.get_instance().match(info=entry)): already.pop() - item_title = entry.get("title") or entry.get("id") or item.url - message = f"Condition '{condition.name}' matched for '{item_title}'." + message = f"Condition '{condition.name}' matched for '{item!r}'." if condition.cli: message += f" Re-queuing with '{condition.cli}'." @@ -873,18 +872,19 @@ class DownloadQueue(metaclass=Singleton): if condition.extras.get("ignore_download", False): extra_msg: str = "" - if yt_conf.get("download_archive") and not condition.extras.get("no_archive", False): - archive_add(yt_conf.get("download_archive"), [archive_id]) - extra_msg = f" and added to archive '{yt_conf.get('download_archive')}'" + if _archive_file and not condition.extras.get("no_archive", False): + archive_add(_archive_file, [archive_id]) + extra_msg = f" and added to archive file '{_archive_file}'" - log_message = f"Ignoring download of '{item_title}' as per condition '{condition.name}'{extra_msg}." + _name = entry.get("title", entry.get("id")) + log_message = f"Ignoring download of '{_name!r}' as per condition '{condition.name}'{extra_msg}." store_type, _ = await self.get_item(archive_id=archive_id) if not store_type: dlInfo = Download( info=ItemDTO( id=entry.get("id"), - title=item_title, + title=_name, url=item.url, preset=item.preset, folder=item.folder, @@ -909,7 +909,7 @@ class DownloadQueue(metaclass=Singleton): if condition.extras.get("set_preset") and (target_preset := condition.extras.get("set_preset")): if Presets.get_instance().has(target_preset): - log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item_title}' as per condition '{condition.name}'." + log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item!r}' as per condition '{condition.name}'." LOG.info(log_message) self._notify.emit(Events.LOG_INFO, data={}, title="Preset Switched", message=log_message) item = item.new_with(preset=target_preset) diff --git a/app/library/Notifications.py b/app/library/Notifications.py index 91a95807..81d7b17a 100644 --- a/app/library/Notifications.py +++ b/app/library/Notifications.py @@ -15,6 +15,7 @@ from .BackgroundWorker import BackgroundWorker from .config import Config from .encoder import Encoder from .Events import Event, EventBus, Events +from .httpx_client import async_client from .ItemDTO import Item, ItemDTO from .Presets import Preset, Presets from .Singleton import Singleton @@ -147,7 +148,7 @@ class Notification(metaclass=Singleton): "Debug mode." self._file: Path = Path(file) if file else Path(config.config_path).joinpath("notifications.json") "File to store notification targets." - self._client: httpx.AsyncClient = client or httpx.AsyncClient() + self._client: httpx.AsyncClient = client or async_client() "HTTP client to send requests." self._encoder: Encoder = encoder or Encoder() "Encoder to encode data." diff --git a/app/library/PackageInstaller.py b/app/library/PackageInstaller.py index 7ad3782e..b2ed929e 100644 --- a/app/library/PackageInstaller.py +++ b/app/library/PackageInstaller.py @@ -6,7 +6,7 @@ import subprocess import sys from pathlib import Path -import httpx +from .httpx_client import sync_client LOG: logging.Logger = logging.getLogger("package_installer") @@ -17,7 +17,7 @@ def parse_version(v: str) -> tuple[int, ...]: class Packages: def __init__(self, env: str | None, file: str | None, upgrade: bool = False): - from_env = env.split() if env else [] + from_env: list[str] = env.split() if env else [] from_file = [] if file: @@ -89,7 +89,7 @@ class PackageInstaller: def _get_latest_version(self, pkg: str) -> str | None: url: str = f"https://pypi.org/pypi/{pkg}/json" try: - with httpx.Client(timeout=5.0) as client: + with sync_client(timeout=5.0) as client: resp = client.get(url) if 200 == resp.status_code: return resp.json()["info"]["version"] diff --git a/app/library/CFSolverRH.py b/app/library/cf_solver_handler.py similarity index 50% rename from app/library/CFSolverRH.py rename to app/library/cf_solver_handler.py index 82822e00..4b23bec4 100644 --- a/app/library/CFSolverRH.py +++ b/app/library/cf_solver_handler.py @@ -1,15 +1,10 @@ -# flake8: noqa: S310 from __future__ import annotations import http.cookiejar -import json import logging -import time -import urllib.request from abc import ABC from collections.abc import Callable from typing import Any, ClassVar -from urllib.parse import urlparse from yt_dlp.networking.common import ( _REQUEST_HANDLERS, @@ -24,15 +19,16 @@ from yt_dlp.networking.common import ( from yt_dlp.networking.exceptions import HTTPError from yt_dlp.utils.networking import clean_headers +from app.library.cf_solver_shared import CACHE, is_cf_challenge, solver + LOG: logging.Logger = logging.getLogger(__name__) SolverFn = Callable[[Request, Response, RequestHandler], Request | None] -CacheEntry = dict[str, Any] def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> Request | None: """ - A Cloudflare solver that uses FlareSolverr/FlareSolverr to solve challenges. + A Cloudflare solver that uses FlareSolverr to solve challenges. Args: request (Request): The original request that triggered the challenge. @@ -43,81 +39,25 @@ def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> Request | None: The modified request with solved credentials, or None if solving failed. """ - from app.library.config import Config - - config = Config.get_instance() - - if not config.flaresolverr_url: + if not isinstance(handler, CFSolverRH): return None - parsed_endpoint = urlparse(config.flaresolverr_url) - if parsed_endpoint.scheme not in ("http", "https"): - return None - - if request.data is not None and request.method not in ("GET", None): - return None - - method: str = request.method.lower() if isinstance(request.method, str) else "get" - if method not in ("get", "head"): - method = "get" - - payload: dict[str, Any] = { - "cmd": f"request.{method}", - "url": request.url, - "maxTimeout": int(getattr(config, "flaresolverr_max_timeout", 60) * 1000), - } - cookiejar = handler._get_cookiejar(request) - cookies = ( - [ - { - "name": cookie.name, - "value": cookie.value, - "domain": cookie.domain or urlparse(request.url).hostname or "", - "path": cookie.path or "/", - } - for cookie in cookiejar - ] - if cookiejar - else [] - ) + host: str = handler._get_host(request.url) + cookies = [ + { + "name": cookie.name, + "value": cookie.value, + "domain": cookie.domain or host, + "path": cookie.path or "/", + } + for cookie in cookiejar + ] - if cookies: - payload["cookies"] = cookies - - req = urllib.request.Request( - config.flaresolverr_url, - data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="POST", - ) - - try: - LOG.info(f"Trying to solve Cloudflare challenge for '{request.url}' this may take a while...") - with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: - result = json.loads(resp.read().decode("utf-8")) - except Exception as e: - LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {e!s}") + solution: dict[str, Any] | None = solver(request.url, cookies, request.headers.get("User-Agent")) + if not handler._apply_solution(solution, request.url, request.headers, cookiejar): return None - if "ok" != result.get("status"): - LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {result}") - return None - - LOG.info(f"Successfully solved Cloudflare challenge for '{request.url}'.") - - solution = result.get("solution") or {} - _cookiejar_from_solution(solution.get("cookies"), request, handler) - - if ua := solution.get("userAgent"): - request.headers["User-Agent"] = ua - - CFSolverRH.cache[urlparse(request.url).netloc] = { - "cookies": solution.get("cookies") or [], - "userAgent": ua, - "expires_at": time.time() + config.flaresolverr_cache_ttl, - } - return request @@ -138,39 +78,6 @@ def set_cf_handler(solver: SolverFn | None = None) -> type[CFSolverRH]: return CFSolverRH -def _cookiejar_from_solution(cookies, request: Request, handler: RequestHandler) -> None: - cookiejar = handler._get_cookiejar(request) - host = urlparse(request.url).hostname or "" - for cookie in cookies or []: - name = cookie.get("name") - value = cookie.get("value") - if not name or value is None: - continue - domain = cookie.get("domain") or host - path = cookie.get("path") or "/" - cookiejar.set_cookie( - http.cookiejar.Cookie( - version=0, - name=name, - value=value, - port=None, - port_specified=False, - domain=domain, - domain_specified=True, - domain_initial_dot=domain.startswith("."), - path=path, - path_specified=True, - secure=bool(cookie.get("secure")), - expires=cookie.get("expires"), - discard=False, - comment=None, - comment_url=None, - rest={}, - rfc2109=False, - ) - ) - - @register_rh class CFSolverRH(RequestHandler, ABC): """Request handler that intercepts Cloudflare challenges""" @@ -178,7 +85,6 @@ class CFSolverRH(RequestHandler, ABC): _SUPPORTED_URL_SCHEMES = ("http", "https") _SUPPORTED_PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h") solver: ClassVar[SolverFn | None] = None - cache: ClassVar[dict[str, CacheEntry]] = {} def __init__(self, *, solver: SolverFn | None = None, **kwargs) -> None: super().__init__(**kwargs) @@ -190,12 +96,66 @@ class CFSolverRH(RequestHandler, ABC): self._fallback_director.close() self._fallback_director = None + @staticmethod + def _get_host(url: str) -> str: + """Extract hostname from URL.""" + from urllib.parse import urlparse + + return urlparse(url).hostname or "" + + def _apply_solution( + self, + solution: dict[str, Any] | None, + url: str, + headers: dict[str, Any], + cookiejar: http.cookiejar.CookieJar, + ) -> bool: + """Apply Cloudflare solution (cookies and User-Agent) to request.""" + if not solution: + return False + + host = self._get_host(url) + + for cookie in solution.get("cookies", []): + name = cookie.get("name") + value = cookie.get("value") + if not name or value is None: + continue + domain = cookie.get("domain") or host + path = cookie.get("path") or "/" + cookiejar.set_cookie( + http.cookiejar.Cookie( + version=0, + name=name, + value=value, + port=None, + port_specified=False, + domain=domain, + domain_specified=True, + domain_initial_dot=domain.startswith("."), + path=path, + path_specified=True, + secure=bool(cookie.get("secure")), + expires=cookie.get("expires"), + discard=False, + comment=None, + comment_url=None, + rest={}, + rfc2109=False, + ) + ) + + if ua := solution.get("userAgent"): + headers["User-Agent"] = ua + + return True + def _check_extensions(self, extensions) -> None: super()._check_extensions(extensions) for key in ("cookiejar", "timeout", "legacy_ssl", "keep_header_casing", "cf_retry"): extensions.pop(key, None) - def _validate(self, request: Request): # type: ignore[override] + def _validate(self, request: Request): self._check_url_scheme(request) self._check_proxies(request.proxies or self.proxies) extensions = request.extensions.copy() @@ -230,30 +190,6 @@ class CFSolverRH(RequestHandler, ABC): self._fallback_director = director return director - @staticmethod - def _is_cf_response(response: Response) -> bool: - """ - Check if the response is a Cloudflare challenge response. - - Args: - response (Response): The HTTP response to check. - - Returns: - bool: True if the response is a Cloudflare challenge, False otherwise. - - """ - status: int | None = getattr(response, "status", None) - if status not in (403, 429, 503): - return False - - headers = response.headers or {} - server_header: str = (headers.get("Server") or "").lower() - if "cloudflare" in server_header: - return True - - cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor") - return any(key in headers for key in cf_header_keys) - def _solve(self, request: Request, response: Response) -> Request | None: return self._solver(request, response, self) if self._solver else None @@ -276,42 +212,32 @@ class CFSolverRH(RequestHandler, ABC): response.close() return director.send(solved_request) - def _inject_cookie(self, request: Request) -> None: - cache_key: str = urlparse(request.url).netloc - if not (cached := self.cache.get(cache_key)): - return - - if cached.get("expires_at", 0) <= time.time(): - self.cache.pop(cache_key, None) - return - - LOG.info(f"Injecting cached Cloudflare cookies for '{cache_key}'.") - _cookiejar_from_solution(cached.get("cookies"), request, self) - if ua := cached.get("userAgent"): - request.headers["User-Agent"] = ua - def _send(self, request: Request) -> Response: - self._inject_cookie(request) + host: str = self._get_host(request.url) + if host and (cached := CACHE.get(host)): + LOG.info(f"Injecting cached Cloudflare cookies for '{host}'.") + self._apply_solution(cached, request.url, request.headers, self._get_cookiejar(request)) + director: RequestDirector = self._build_fallback() try: response: Response = director.send(request) except HTTPError as error: - if error.response and self._is_cf_response(error.response): + if error.response and is_cf_challenge(getattr(error.response, "status", None), error.response.headers): return self._retry_with_clearance(request, error.response, director) raise - if self._is_cf_response(response): + if is_cf_challenge(getattr(response, "status", None), response.headers): return self._retry_with_clearance(request, response, director) return response @register_preference(CFSolverRH) -def cf_solver_preference(_handler: RequestHandler, _request: Request) -> int: +def cf_solver_preference(_handler, _request) -> int: from app.library.config import Config if not Config.get_instance().flaresolverr_url: return 0 - return 100 + return 500 diff --git a/app/library/cf_solver_shared.py b/app/library/cf_solver_shared.py new file mode 100644 index 00000000..b95e0029 --- /dev/null +++ b/app/library/cf_solver_shared.py @@ -0,0 +1,100 @@ +# flake8: noqa: S310 +from __future__ import annotations + +import json +import logging +import time +import urllib.request +from typing import Any +from urllib.parse import urlparse + +from .cache import Cache + +CACHE: Cache = Cache() +LOG: logging.Logger = logging.getLogger("cf_solver") + + +def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> dict[str, Any] | None: + """ + Run FlareSolverr solve. Returns solution dict or None. + + Args: + url (str): The URL to solve the challenge for. + cookies (list[dict]): List of existing cookies to send to FlareSolverr. + user_agent (str | None): The User-Agent string to send to FlareSolverr. + + Returns: + dict[str, Any] | None: The solution dict from FlareSolverr, or None if solving fails. + + """ + from app.library.config import Config + + config = Config.get_instance() + if not (endpoint := config.flaresolverr_url): + return None + + parsed = urlparse(endpoint) + if parsed.scheme not in ("http", "https"): + return None + + host = urlparse(url).hostname or "" + if not host: + return None + + if cached := CACHE.get(host): + return cached + + payload: dict[str, Any] = { + "cmd": "request.get", + "url": url, + "maxTimeout": int(config.flaresolverr_max_timeout * 1000), + } + + if cookies: + payload["cookies"] = cookies + + if user_agent: + payload.setdefault("headers", {})["User-Agent"] = user_agent + + req = urllib.request.Request( + endpoint, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + + LOG.info(f"Solving Cloudflare challenge for '{host}' via FlareSolverr.") + start_time = time.time() + with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: + result = json.loads(resp.read().decode("utf-8")) + + if "ok" != result.get("status"): + LOG.error(f"FlareSolverr failed to solve challenge for '{host}': {result.get('message')}") + return None + + LOG.info(f"FlareSolverr successfully solved challenge for '{host}'. (Took {time.time() - start_time:.2f} seconds)") + + solution = result.get("solution") or {} + CACHE.set( + host, + {"cookies": solution.get("cookies") or [], "userAgent": solution.get("userAgent")}, + ttl=config.flaresolverr_cache_ttl, + ) + + return CACHE.get(host) + + +def is_cf_challenge(status: int | None, headers: dict[str, Any] | None) -> bool: + """ + Determine whether a response indicates a Cloudflare challenge. + """ + if status not in (403, 429, 503): + return False + + headers = headers or {} + server_header: str = str(headers.get("Server", "")).lower() + if "cloudflare" in server_header: + return True + + cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor") + return any(key in headers for key in cf_header_keys) diff --git a/app/library/httpx_client.py b/app/library/httpx_client.py new file mode 100644 index 00000000..78fd3a1b --- /dev/null +++ b/app/library/httpx_client.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import logging +from typing import Any + +import httpx + +from .cf_solver_shared import is_cf_challenge, solver + +__all__: list[str] = ["async_client", "sync_client"] + +LOG: logging.Logger = logging.getLogger("httpx_cf") + + +def _parse_cookie_header(cookie_header: str | None) -> dict[str, str]: + cookies: dict[str, str] = {} + + if not cookie_header: + return cookies + + for item in cookie_header.split(";"): + item: str = item.strip() + if "=" in item: + name, value = item.split("=", 1) + cookies[name] = value + + return cookies + + +def _merge_cookies(existing_header: str | None, new_cookies: list[dict[str, str]]) -> dict[str, str]: + merged: dict[str, str] = _parse_cookie_header(existing_header) + + for cookie in new_cookies: + if cookie.get("name") and cookie.get("value"): + merged[cookie["name"]] = cookie["value"] + + return merged + + +def _get_transport( + enable_cf: bool, + is_async: bool, + transport: httpx.AsyncBaseTransport | None, +) -> httpx.AsyncBaseTransport: + if enable_cf: + return CFAsyncTransport(base=transport) if is_async else CFTransport(base=transport) + + return transport or (httpx.AsyncHTTPTransport() if is_async else httpx.HTTPTransport()) + + +class CFAsyncTransport(httpx.AsyncBaseTransport): + def __init__(self, base: httpx.AsyncBaseTransport | None = None): + self.base: httpx.AsyncBaseTransport | httpx.AsyncHTTPTransport = base or httpx.AsyncHTTPTransport() + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + response: httpx.Response = await self.base.handle_async_request(request) + if not is_cf_challenge(response.status_code, dict(response.headers)): + return response + + url = str(request.url) + + solution: dict[str, Any] | None = solver(url, [], request.headers.get("User-Agent")) + if not solution: + return response + + if cookies := solution.get("cookies", []): + merged_cookies: dict[str, str] = _merge_cookies(request.headers.get("Cookie"), cookies) + headers: httpx.Headers = request.headers.copy() + headers.pop("Cookie", None) + request = httpx.Request( + method=request.method, + url=request.url, + headers=headers, + content=request.content, + cookies=merged_cookies, + ) + + if ua := solution.get("userAgent"): + request.headers["User-Agent"] = ua + + await response.aclose() + return await self.base.handle_async_request(request) + + def close(self) -> None: + self.base.close() + + +class CFTransport(httpx.BaseTransport): + def __init__(self, base: httpx.BaseTransport | None = None): + self.base: httpx.BaseTransport | httpx.HTTPTransport = base or httpx.HTTPTransport() + + def handle_request(self, request: httpx.Request) -> httpx.Response: + response: httpx.Response = self.base.handle_request(request) + if not is_cf_challenge(response.status_code, dict(response.headers)): + return response + + url = str(request.url) + + solution: dict[str, Any] | None = solver(url, [], request.headers.get("User-Agent")) + if not solution: + return response + + if cookies := solution.get("cookies", []): + merged_cookies: dict[str, str] = _merge_cookies(request.headers.get("Cookie"), cookies) + headers: httpx.Headers = request.headers.copy() + headers.pop("Cookie", None) + request = httpx.Request( + method=request.method, + url=request.url, + headers=headers, + content=request.content, + cookies=merged_cookies, + ) + + if ua := solution.get("userAgent"): + request.headers["User-Agent"] = ua + + response.close() + return self.base.handle_request(request) + + def close(self) -> None: + self.base.close() + + +def async_client(enable_cf: bool = True, **kwargs: Any) -> httpx.AsyncClient: + """ + Create an httpx.AsyncClient with optional Cloudflare challenge solving. + + Args: + enable_cf (bool): Whether to enable Cloudflare challenge solving. + **kwargs: Additional keyword arguments to pass to httpx.AsyncClient. + + Returns: + httpx.AsyncClient: The configured httpx.AsyncClient instance. + + """ + transport = kwargs.pop("transport", None) + return httpx.AsyncClient(transport=_get_transport(enable_cf, is_async=True, transport=transport), **kwargs) + + +def sync_client(enable_cf: bool = True, **kwargs: Any) -> httpx.Client: + """ + Create an httpx.Client with optional Cloudflare challenge solving. + + Args: + enable_cf (bool): Whether to enable Cloudflare challenge solving. + **kwargs: Additional keyword arguments to pass to httpx.Client. + + Returns: + httpx.Client: The configured httpx.Client instance. + + """ + transport = kwargs.pop("transport", None) + return httpx.Client(transport=_get_transport(enable_cf, is_async=False, transport=transport), **kwargs) diff --git a/app/library/task_handlers/_base_handler.py b/app/library/task_handlers/_base_handler.py index fbb663da..bbf3c32f 100644 --- a/app/library/task_handlers/_base_handler.py +++ b/app/library/task_handlers/_base_handler.py @@ -5,6 +5,7 @@ import httpx from yt_dlp.utils.networking import random_user_agent from app.library.config import Config +from app.library.httpx_client import async_client from app.library.Tasks import Task, TaskFailure, TaskResult @@ -73,7 +74,7 @@ class BaseHandler: if proxy := ytdlp_opts.get("proxy", None): opts["proxy"] = proxy - async with httpx.AsyncClient(**opts) as client: + async with async_client(**opts) as client: method = kwargs.pop("method", "GET").upper() timeout = ytdlp_opts.get("timeout", ytdlp_opts.get("socket_timeout", 120)) return await client.request(method=method, url=url, timeout=timeout, **kwargs) diff --git a/app/library/task_handlers/generic.py b/app/library/task_handlers/generic.py index cc6ce5ab..24f1dc56 100644 --- a/app/library/task_handlers/generic.py +++ b/app/library/task_handlers/generic.py @@ -14,7 +14,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Literal from urllib.parse import urljoin -import httpx import jmespath from parsel import Selector from parsel.selector import SelectorList @@ -22,12 +21,14 @@ from yt_dlp.utils.networking import random_user_agent from app.library.cache import Cache from app.library.config import Config +from app.library.httpx_client import async_client from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult from app.library.Utils import extract_info, get_archive_id from ._base_handler import BaseHandler if TYPE_CHECKING: + import httpx from parsel.selector import SelectorList LOG: logging.Logger = logging.getLogger(__name__) @@ -474,7 +475,7 @@ def load_task_definitions(config: Config | None = None) -> list[TaskDefinition]: LOG.error(f"[{path.name}] Unsupported response type '{response_type}'.") continue - response_config = ResponseConfig(format=response_type) # type: ignore[arg-type] + response_config = ResponseConfig(format=response_type) parse_raw: Mapping | None = raw.get("parse") if not isinstance(parse_raw, Mapping): @@ -826,7 +827,7 @@ class GenericTaskHandler(BaseHandler): timeout_value: float | Any = definition.request.timeout or ytdlp_opts.get("socket_timeout", 120) - async with httpx.AsyncClient(**client_options) as client: + async with async_client(**client_options) as client: response: httpx.Response = await client.request( method=definition.request.normalized_method(), url=url, diff --git a/app/library/ytdlp.py b/app/library/ytdlp.py index 6e95166c..3b56f7b3 100644 --- a/app/library/ytdlp.py +++ b/app/library/ytdlp.py @@ -1,11 +1,11 @@ # flake8: noqa: F401 +import sys from typing import Any import yt_dlp from yt_dlp.utils import make_archive_id -# imported to register the handler with yt-dlp. -from app.library.CFSolverRH import set_cf_handler +from app.library.cf_solver_handler import set_cf_handler class _ArchiveProxy: diff --git a/app/routes/api/docs.py b/app/routes/api/docs.py index e4416321..f377f515 100644 --- a/app/routes/api/docs.py +++ b/app/routes/api/docs.py @@ -3,7 +3,6 @@ import time from datetime import UTC, datetime from typing import Any -import httpx from aiohttp import web from aiohttp.web import Request, Response from yt_dlp.utils.networking import random_user_agent @@ -86,7 +85,9 @@ async def get_doc(request: Request, config: Config, cache: Cache) -> Response: except Exception: pass - async with httpx.AsyncClient(**opts) as client: + from app.library.httpx_client import async_client + + async with async_client(**opts) as client: LOG.debug(f"Fetching doc from '{url}'.") response = await client.request(method="GET", url=url, follow_redirects=True) dct = { diff --git a/app/routes/api/images.py b/app/routes/api/images.py index aa73cb17..8cfcc74a 100644 --- a/app/routes/api/images.py +++ b/app/routes/api/images.py @@ -5,7 +5,6 @@ from datetime import UTC, datetime from typing import Any from urllib.parse import urlparse -import httpx from aiohttp import web from aiohttp.web import Request, Response from yt_dlp.utils.networking import random_user_agent @@ -67,7 +66,9 @@ async def get_thumbnail(request: Request, config: Config) -> Response: except Exception: pass - async with httpx.AsyncClient(**opts) as client: + from app.library.httpx_client import async_client + + async with async_client(**opts) as client: LOG.debug(f"Fetching thumbnail from '{url}'.") response = await client.request(method="GET", url=url, follow_redirects=True) @@ -158,7 +159,9 @@ async def get_background(request: Request, config: Config, cache: Cache) -> Resp except Exception: pass - async with httpx.AsyncClient(**opts) as client: + from app.library.httpx_client import async_client + + async with async_client(**opts) as client: if backend.startswith("https://www.bing.com/HPImageArchive.aspx"): if not cache.has(CACHE_KEY_BING): response = await client.request(method="GET", url=backend) diff --git a/app/routes/api/tasks.py b/app/routes/api/tasks.py index e6de60d9..b0848484 100644 --- a/app/routes/api/tasks.py +++ b/app/routes/api/tasks.py @@ -307,9 +307,10 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R filename.write_text(xml_content, encoding="utf-8") try: - import httpx from yt_dlp.utils.networking import random_user_agent + from app.library.httpx_client import async_client + ytdlp_args: dict = task.get_ytdlp_opts().get_all() opts: dict[str, Any] = { "headers": { @@ -331,7 +332,7 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R except Exception: pass - async with httpx.AsyncClient(**opts) as client: + async with async_client(**opts) as client: for key in info.get("thumbnails", {}): try: url = info["thumbnails"][key] diff --git a/app/tests/test_cf_solver_handler.py b/app/tests/test_cf_solver_handler.py new file mode 100644 index 00000000..4d14275e --- /dev/null +++ b/app/tests/test_cf_solver_handler.py @@ -0,0 +1,257 @@ +"""Comprehensive tests for cf_solver_handler yt-dlp request handler.""" + +from __future__ import annotations + +import http.cookiejar +from unittest.mock import Mock, patch + +import pytest + +try: + from yt_dlp.networking.common import Request, RequestDirector, Response + from yt_dlp.networking.exceptions import HTTPError + + YTDLP_AVAILABLE = True +except ImportError: + YTDLP_AVAILABLE = False + Request = Mock + Response = Mock + RequestDirector = Mock + HTTPError = Exception + +pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available") + + +@pytest.fixture(scope="module") +def cf_handler_module(): + """Lazily import cf_solver_handler module to avoid circular imports.""" + from app.library import cf_solver_handler + + return cf_solver_handler + + +class TestCfSolverFunction: + """Test the cf_solver function.""" + + @patch("app.library.cf_solver_handler.solver") + def test_cf_solver_success(self, mock_solver, cf_handler_module): + """Test successful solving of Cloudflare challenge.""" + mock_solver.return_value = {"cookies": [], "userAgent": "Mozilla/5.0"} + + handler = cf_handler_module.CFSolverRH(logger=Mock()) + cookiejar = http.cookiejar.CookieJar() + handler._get_cookiejar = Mock(return_value=cookiejar) + + request = Mock() + request.url = "https://example.com/path" + request.headers = {"User-Agent": "test"} + + response = Mock() + + result = cf_handler_module.cf_solver(request, response, handler) + + assert result is request + mock_solver.assert_called_once() + + @patch("app.library.cf_solver_handler.solver") + def test_cf_solver_no_solution(self, mock_solver, cf_handler_module): + """Test when solver returns no solution.""" + mock_solver.return_value = None + + handler = cf_handler_module.CFSolverRH(logger=Mock()) + cookiejar = http.cookiejar.CookieJar() + handler._get_cookiejar = Mock(return_value=cookiejar) + + request = Mock() + request.url = "https://example.com/path" + request.headers = {"User-Agent": "test"} + + response = Mock() + + result = cf_handler_module.cf_solver(request, response, handler) + + assert result is None + mock_solver.assert_called_once() + + @patch("app.library.cf_solver_handler.solver") + def test_cf_solver_with_existing_cookies(self, mock_solver, cf_handler_module): + """Test solving with existing cookies in jar.""" + mock_solver.return_value = {"cookies": [], "userAgent": "Mozilla/5.0"} + + handler = cf_handler_module.CFSolverRH(logger=Mock()) + cookiejar = http.cookiejar.CookieJar() + + cookie = http.cookiejar.Cookie( + version=0, + name="existing", + value="value", + port=None, + port_specified=False, + domain="example.com", + domain_specified=True, + domain_initial_dot=False, + path="/", + path_specified=True, + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={}, + rfc2109=False, + ) + cookiejar.set_cookie(cookie) + handler._get_cookiejar = Mock(return_value=cookiejar) + + request = Mock() + request.url = "https://example.com/path" + request.headers = {"User-Agent": "test"} + + response = Mock() + + result = cf_handler_module.cf_solver(request, response, handler) + + assert result is request + call_args = mock_solver.call_args + assert call_args is not None + cookies_arg = call_args[0][1] + assert len(cookies_arg) > 0 + assert "existing" == cookies_arg[0]["name"] + + +class TestSetCfHandler: + """Test set_cf_handler function.""" + + def test_set_cf_handler_default(self, cf_handler_module): + """Test setting CF handler with default solver.""" + result = cf_handler_module.set_cf_handler() + assert result is cf_handler_module.CFSolverRH + assert cf_handler_module.CFSolverRH.solver is None or callable(cf_handler_module.CFSolverRH.solver) + + def test_set_cf_handler_custom_solver(self, cf_handler_module): + """Test setting CF handler with custom solver.""" + + def custom_solver(req, resp, handler): + return req + + result = cf_handler_module.set_cf_handler(custom_solver) + assert result is cf_handler_module.CFSolverRH + assert cf_handler_module.CFSolverRH.solver is custom_solver + + +class TestCFSolverRH: + """Test CFSolverRH request handler class.""" + + @pytest.fixture(autouse=True) + def setup(self, cf_handler_module): + """Set up test fixtures.""" + self.module = cf_handler_module + self.handler = self.module.CFSolverRH(logger=Mock(), verbose=False) + + def test_init_default(self): + """Test initialization with defaults.""" + handler = self.module.CFSolverRH(logger=Mock(), verbose=False) + assert handler._solver is not None + assert handler._fallback_director is None + + def test_init_custom_solver(self): + """Test initialization with custom solver.""" + + def custom_solver(req, resp, handler): + return req + + handler = self.module.CFSolverRH(logger=Mock(), verbose=False, solver=custom_solver) + assert handler._solver is custom_solver + + def test_close(self): + """Test closing handler.""" + mock_director = Mock() + self.handler._fallback_director = mock_director + + self.handler.close() + + mock_director.close.assert_called_once() + assert self.handler._fallback_director is None + + def test_close_no_director(self): + """Test closing handler when no director exists.""" + self.handler._fallback_director = None + self.handler.close() + + def test_check_extensions(self): + """Test extension checking.""" + extensions = { + "timeout": 30, + "legacy_ssl": True, + "other": "value", + } + + self.handler._check_extensions(extensions) + + def test_validate(self): + """Test request validation.""" + request = Mock() + request.url = "https://example.com" + request.proxies = None + request.extensions = {} + + self.handler._validate(request) + + def test_solve(self): + """Test solving challenge.""" + request = Mock() + request.url = "https://example.com" + request.headers = {} + request.extensions = {} + + response = Mock() + + self.handler._solver = Mock(return_value=request) + result = self.handler._solve(request, response) + assert result is request + self.handler._solver.assert_called_once() + + self.handler._solver = None + result = self.handler._solve(request, response) + assert result is None + + def test_mark_retry(self): + """Test marking request as retry.""" + request = Mock() + request.copy = Mock(return_value=Mock()) + request.copy.return_value.extensions = {} + + new_request = self.module.CFSolverRH._mark_retry(request) + + assert new_request.extensions.get("cf_retry") is True + + +class TestCfSolverPreference: + """Test cf_solver_preference function.""" + + def test_preference_with_flaresolverr(self, cf_handler_module, monkeypatch): + """Test preference when FlareSolverr is configured.""" + mock_config = Mock() + mock_config.flaresolverr_url = "http://localhost:8191/v1" + + import app.library.config + + monkeypatch.setattr(app.library.config.Config, "get_instance", lambda: mock_config) + + def test_preference_without_flaresolverr(self, cf_handler_module, monkeypatch): + """Test preference when FlareSolverr is not configured.""" + mock_config = Mock() + mock_config.flaresolverr_url = None + + import app.library.config + + monkeypatch.setattr(app.library.config.Config, "get_instance", lambda: mock_config) + + def test_preference_with_empty_flaresolverr(self, cf_handler_module, monkeypatch): + """Test preference when FlareSolverr URL is empty.""" + mock_config = Mock() + mock_config.flaresolverr_url = "" + + import app.library.config + + monkeypatch.setattr(app.library.config.Config, "get_instance", lambda: mock_config) diff --git a/app/tests/test_httpx_client.py b/app/tests/test_httpx_client.py new file mode 100644 index 00000000..705a946e --- /dev/null +++ b/app/tests/test_httpx_client.py @@ -0,0 +1,436 @@ +"""Comprehensive tests for httpx_client Cloudflare challenge solving.""" + +from __future__ import annotations + +import http.cookiejar +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +import httpx +import pytest + +from app.library.httpx_client import ( + CFAsyncTransport, + CFTransport, + _get_transport, + async_client, + sync_client, +) + + +class TestGetTransport: + """Test transport factory function.""" + + def test_get_transport_cf_enabled_async(self): + """Test getting async transport with CF enabled.""" + transport = _get_transport(enable_cf=True, is_async=True, transport=None) + assert isinstance(transport, CFAsyncTransport) + + def test_get_transport_cf_enabled_sync(self): + """Test getting sync transport with CF enabled.""" + transport = _get_transport(enable_cf=False, is_async=False, transport=None) + assert isinstance(transport, httpx.HTTPTransport) + + def test_get_transport_cf_disabled_async(self): + """Test getting async transport with CF disabled.""" + transport = _get_transport(enable_cf=False, is_async=True, transport=None) + assert isinstance(transport, httpx.AsyncHTTPTransport) + + def test_get_transport_cf_disabled_sync(self): + """Test getting sync transport with CF disabled.""" + transport = _get_transport(enable_cf=False, is_async=False, transport=None) + assert isinstance(transport, httpx.HTTPTransport) + + def test_get_transport_custom_base_async(self): + """Test getting transport with custom base transport.""" + custom_transport = httpx.AsyncHTTPTransport() + transport = _get_transport(enable_cf=True, is_async=True, transport=custom_transport) + assert isinstance(transport, CFAsyncTransport) + assert transport.base is custom_transport + + def test_get_transport_custom_base_sync(self): + """Test getting transport with custom base transport.""" + custom_transport = httpx.HTTPTransport() + transport = _get_transport(enable_cf=True, is_async=False, transport=custom_transport) + assert isinstance(transport, CFTransport) + assert transport.base is custom_transport + + +class TestCFAsyncTransport: + """Test async Cloudflare transport wrapper.""" + + def setup_method(self): + """Set up test fixtures.""" + self.base_transport = AsyncMock(spec=httpx.AsyncHTTPTransport) + self.transport = CFAsyncTransport(base=self.base_transport) + + @pytest.mark.asyncio + async def test_init_default(self): + """Test initialization with defaults.""" + transport = CFAsyncTransport() + assert isinstance(transport.base, httpx.AsyncHTTPTransport) + + @pytest.mark.asyncio + async def test_init_custom(self): + """Test initialization with custom base.""" + assert self.transport.base is self.base_transport + + @pytest.mark.asyncio + @patch("app.library.httpx_client.is_cf_challenge") + async def test_handle_request_no_challenge(self, mock_is_cf): + """Test handling request without Cloudflare challenge.""" + mock_is_cf.return_value = False + + request = httpx.Request("GET", "https://example.com") + response = Mock(status_code=200, headers={}) + self.base_transport.handle_async_request.return_value = response + + result = await self.transport.handle_async_request(request) + + assert result is response + self.base_transport.handle_async_request.assert_called_once() + mock_is_cf.assert_called_once_with(200, {}) + + @pytest.mark.asyncio + @pytest.mark.asyncio + @patch("app.library.httpx_client.solver") + @patch("app.library.httpx_client.is_cf_challenge") + async def test_handle_request_cached_solution(self, mock_is_cf, mock_solver): + """Test handling request with Cloudflare solution from cache.""" + mock_is_cf.return_value = True + mock_solver.return_value = { + "cookies": [{"name": "cf_clearance", "value": "token123"}], + "userAgent": "Mozilla/5.0", + } + + request = httpx.Request("GET", "https://example.com") + response1 = Mock(status_code=403, headers={"Server": "cloudflare"}) + response1.aclose = AsyncMock() + response2 = Mock(status_code=200, headers={}) + + self.base_transport.handle_async_request.side_effect = [response1, response2] + + result = await self.transport.handle_async_request(request) + + assert result is response2 + assert 2 == self.base_transport.handle_async_request.call_count + mock_solver.assert_called_once() + response1.aclose.assert_called_once() + + second_call_request = self.base_transport.handle_async_request.call_args_list[1][0][0] + assert "cf_clearance=token123" == second_call_request.headers["Cookie"] + assert "Mozilla/5.0" == second_call_request.headers["User-Agent"] + + @pytest.mark.asyncio + @patch("app.library.httpx_client.solver") + @patch("app.library.httpx_client.is_cf_challenge") + async def test_handle_request_solve_challenge(self, mock_is_cf, mock_solver): + """Test handling request by solving Cloudflare challenge.""" + mock_is_cf.return_value = True + mock_solver.return_value = { + "cookies": [{"name": "cf_clearance", "value": "token123"}, {"name": "__cf_bm", "value": "bm456"}], + "userAgent": "Mozilla/5.0", + } + + request = httpx.Request("GET", "https://example.com") + response1 = Mock(status_code=403, headers={"cf-ray": "123"}) + response1.aclose = AsyncMock() + response2 = Mock(status_code=200, headers={}) + + self.base_transport.handle_async_request.side_effect = [response1, response2] + + result = await self.transport.handle_async_request(request) + + assert result is response2 + assert 2 == self.base_transport.handle_async_request.call_count + mock_solver.assert_called_once() + response1.aclose.assert_called_once() + + second_call_request = self.base_transport.handle_async_request.call_args_list[1][0][0] + assert "cf_clearance=token123; __cf_bm=bm456" == second_call_request.headers["Cookie"] + assert "Mozilla/5.0" == second_call_request.headers["User-Agent"] + + @pytest.mark.asyncio + @patch("app.library.httpx_client.solver") + @patch("app.library.httpx_client.is_cf_challenge") + async def test_handle_request_merge_existing_cookies(self, mock_is_cf, mock_solver): + """Test that existing cookies are preserved when adding CF cookies.""" + mock_is_cf.return_value = True + mock_solver.return_value = { + "cookies": [{"name": "cf_clearance", "value": "token123"}], + "userAgent": "Mozilla/5.0", + } + + request = httpx.Request("GET", "https://example.com", cookies={"session_id": "abc123", "user_pref": "dark"}) + response1 = Mock(status_code=403, headers={"cf-ray": "123"}) + response1.aclose = AsyncMock() + response2 = Mock(status_code=200, headers={}) + + self.base_transport.handle_async_request.side_effect = [response1, response2] + + result = await self.transport.handle_async_request(request) + + assert result is response2 + + second_call_request = self.base_transport.handle_async_request.call_args_list[1][0][0] + cookie_header = second_call_request.headers["Cookie"] + assert "session_id=abc123" in cookie_header + assert "user_pref=dark" in cookie_header + assert "cf_clearance=token123" in cookie_header + + @pytest.mark.asyncio + @patch("app.library.httpx_client.solver") + @patch("app.library.httpx_client.is_cf_challenge") + async def test_handle_request_solve_failed(self, mock_is_cf, mock_solver): + """Test handling request when solving fails.""" + mock_is_cf.return_value = True + mock_solver.return_value = None + + request = httpx.Request("GET", "https://example.com") + response = Mock(status_code=403, headers={"cf-ray": "123"}) + response.close = Mock() + + self.base_transport.handle_async_request.return_value = response + + result = await self.transport.handle_async_request(request) + + assert result is response + assert 1 == self.base_transport.handle_async_request.call_count + mock_solver.assert_called_once() + response.close.assert_not_called() + + def test_close(self): + """Test closing transport.""" + self.base_transport.close = Mock() + self.transport.close() + self.base_transport.close.assert_called_once() + + +class TestCFTransport: + """Test sync Cloudflare transport wrapper.""" + + def setup_method(self): + """Set up test fixtures.""" + self.base_transport = Mock(spec=httpx.HTTPTransport) + self.transport = CFTransport(base=self.base_transport) + + def test_init_default(self): + """Test initialization with defaults.""" + transport = CFTransport() + assert isinstance(transport.base, httpx.HTTPTransport) + + def test_init_custom(self): + """Test initialization with custom base.""" + assert self.transport.base is self.base_transport + + @patch("app.library.httpx_client.is_cf_challenge") + def test_handle_request_no_challenge(self, mock_is_cf): + """Test handling request without Cloudflare challenge.""" + mock_is_cf.return_value = False + + request = httpx.Request("GET", "https://example.com") + response = Mock(status_code=200, headers={}) + self.base_transport.handle_request.return_value = response + + result = self.transport.handle_request(request) + + assert result is response + self.base_transport.handle_request.assert_called_once() + mock_is_cf.assert_called_once_with(200, {}) + + @patch("app.library.httpx_client.solver") + @patch("app.library.httpx_client.is_cf_challenge") + def test_handle_request_cached_solution(self, mock_is_cf, mock_solver): + """Test handling request with Cloudflare solution from cache.""" + mock_is_cf.return_value = True + mock_solver.return_value = { + "cookies": [{"name": "cf_clearance", "value": "token123"}], + "userAgent": "Mozilla/5.0", + } + + request = httpx.Request("GET", "https://example.com") + response1 = Mock(status_code=403, headers={"Server": "cloudflare"}) + response1.close = Mock() + response2 = Mock(status_code=200, headers={}) + + self.base_transport.handle_request.side_effect = [response1, response2] + + result = self.transport.handle_request(request) + + assert result is response2 + assert 2 == self.base_transport.handle_request.call_count + mock_solver.assert_called_once() + response1.close.assert_called_once() + + second_call_request = self.base_transport.handle_request.call_args_list[1][0][0] + assert "cf_clearance=token123" == second_call_request.headers["Cookie"] + assert "Mozilla/5.0" == second_call_request.headers["User-Agent"] + + @patch("app.library.httpx_client.solver") + @patch("app.library.httpx_client.is_cf_challenge") + def test_handle_request_solve_challenge(self, mock_is_cf, mock_solver): + """Test handling request by solving Cloudflare challenge.""" + mock_is_cf.return_value = True + mock_solver.return_value = { + "cookies": [{"name": "cf_clearance", "value": "token123"}, {"name": "__cf_bm", "value": "bm456"}], + "userAgent": "Mozilla/5.0", + } + + request = httpx.Request("GET", "https://example.com") + response1 = Mock(status_code=503, headers={"cf-cache-status": "DYNAMIC"}) + response1.close = Mock() + response2 = Mock(status_code=200, headers={}) + + self.base_transport.handle_request.side_effect = [response1, response2] + + result = self.transport.handle_request(request) + + assert result is response2 + assert 2 == self.base_transport.handle_request.call_count + mock_solver.assert_called_once() + response1.close.assert_called_once() + + second_call_request = self.base_transport.handle_request.call_args_list[1][0][0] + assert "cf_clearance=token123; __cf_bm=bm456" == second_call_request.headers["Cookie"] + assert "Mozilla/5.0" == second_call_request.headers["User-Agent"] + + @patch("app.library.httpx_client.solver") + @patch("app.library.httpx_client.is_cf_challenge") + def test_handle_request_merge_existing_cookies(self, mock_is_cf, mock_solver): + """Test that existing cookies are preserved when adding CF cookies.""" + mock_is_cf.return_value = True + mock_solver.return_value = { + "cookies": [{"name": "cf_clearance", "value": "token123"}], + "userAgent": "Mozilla/5.0", + } + + request = httpx.Request("GET", "https://example.com", cookies={"session_id": "abc123", "user_pref": "dark"}) + response1 = Mock(status_code=403, headers={"cf-ray": "123"}) + response1.close = Mock() + response2 = Mock(status_code=200, headers={}) + + self.base_transport.handle_request.side_effect = [response1, response2] + + result = self.transport.handle_request(request) + + assert result is response2 + + second_call_request = self.base_transport.handle_request.call_args_list[1][0][0] + cookie_header = second_call_request.headers["Cookie"] + assert "session_id=abc123" in cookie_header + assert "user_pref=dark" in cookie_header + assert "cf_clearance=token123" in cookie_header + + @patch("app.library.httpx_client.solver") + @patch("app.library.httpx_client.is_cf_challenge") + def test_handle_request_solve_failed(self, mock_is_cf, mock_solver): + """Test handling request when solving fails.""" + mock_is_cf.return_value = True + mock_solver.return_value = None + + request = httpx.Request("GET", "https://example.com") + response = Mock(status_code=429, headers={"cf-ray": "abc123"}) + response.close = Mock() + + self.base_transport.handle_request.return_value = response + + result = self.transport.handle_request(request) + + assert result is response + assert 1 == self.base_transport.handle_request.call_count + mock_solver.assert_called_once() + response.close.assert_not_called() + + def test_close(self): + """Test closing transport.""" + self.base_transport.close = Mock() + self.transport.close() + self.base_transport.close.assert_called_once() + + +class TestAsyncClient: + """Test async client factory function.""" + + @pytest.mark.asyncio + async def test_async_client_cf_enabled(self): + """Test creating async client with CF enabled.""" + async with async_client(enable_cf=True) as client: + assert isinstance(client, httpx.AsyncClient) + assert isinstance(client._transport, CFAsyncTransport) + + @pytest.mark.asyncio + async def test_async_client_cf_disabled(self): + """Test creating async client with CF disabled.""" + async with async_client(enable_cf=False) as client: + assert isinstance(client, httpx.AsyncClient) + assert isinstance(client._transport, httpx.AsyncHTTPTransport) + + @pytest.mark.asyncio + async def test_async_client_kwargs(self): + """Test creating async client with additional kwargs.""" + async with async_client(enable_cf=True, timeout=30.0, follow_redirects=True) as client: + assert isinstance(client, httpx.AsyncClient) + assert isinstance(client._transport, CFAsyncTransport) + assert 30.0 == client.timeout.read + assert client.follow_redirects is True + + @pytest.mark.asyncio + async def test_async_client_custom_transport(self): + """Test creating async client with custom transport.""" + custom = httpx.AsyncHTTPTransport() + async with async_client(enable_cf=True, transport=custom) as client: + assert isinstance(client, httpx.AsyncClient) + assert isinstance(client._transport, CFAsyncTransport) + assert client._transport.base is custom + + @pytest.mark.asyncio + async def test_async_client_custom_transport_cf_disabled(self): + """Test creating async client with custom transport and CF disabled.""" + custom = httpx.AsyncHTTPTransport() + async with async_client(enable_cf=False, transport=custom) as client: + assert isinstance(client, httpx.AsyncClient) + + assert client._transport is custom + + +class TestSyncClient: + """Test sync client factory function.""" + + def test_sync_client_cf_enabled(self): + """Test creating sync client with CF enabled.""" + client = sync_client(enable_cf=True) + assert isinstance(client, httpx.Client) + assert isinstance(client._transport, CFTransport) + client.close() + + def test_sync_client_cf_disabled(self): + """Test creating sync client with CF disabled.""" + client = sync_client(enable_cf=False) + assert isinstance(client, httpx.Client) + assert isinstance(client._transport, httpx.HTTPTransport) + client.close() + + def test_sync_client_kwargs(self): + """Test creating sync client with additional kwargs.""" + client = sync_client(enable_cf=True, timeout=30.0, follow_redirects=True) + assert isinstance(client, httpx.Client) + assert isinstance(client._transport, CFTransport) + assert 30.0 == client.timeout.read + assert client.follow_redirects is True + client.close() + + def test_sync_client_custom_transport(self): + """Test creating sync client with custom transport.""" + custom = httpx.HTTPTransport() + client = sync_client(enable_cf=True, transport=custom) + assert isinstance(client, httpx.Client) + assert isinstance(client._transport, CFTransport) + assert client._transport.base is custom + client.close() + + def test_sync_client_custom_transport_cf_disabled(self): + """Test creating sync client with custom transport and CF disabled.""" + custom = httpx.HTTPTransport() + client = sync_client(enable_cf=False, transport=custom) + assert isinstance(client, httpx.Client) + assert client._transport is custom, "When CF is disabled, custom transport should be used directly" + client.close() diff --git a/app/tests/test_package_installer.py b/app/tests/test_package_installer.py index 6c71ed87..e07b736a 100644 --- a/app/tests/test_package_installer.py +++ b/app/tests/test_package_installer.py @@ -95,7 +95,7 @@ class TestInstalledAndLatest: mock_version.side_effect = PackageNotFoundError assert inst._get_installed_version("bar") is None - @patch("app.library.PackageInstaller.httpx.Client") + @patch("app.library.PackageInstaller.sync_client") def test_get_latest_version_success(self, mock_client, tmp_path: Path) -> None: inst = PackageInstaller(pkg_path=tmp_path) @@ -108,7 +108,7 @@ class TestInstalledAndLatest: assert inst._get_latest_version("foo") == "9.9.9" - @patch("app.library.PackageInstaller.httpx.Client") + @patch("app.library.PackageInstaller.sync_client") def test_get_latest_version_non_200(self, mock_client, tmp_path: Path) -> None: inst = PackageInstaller(pkg_path=tmp_path) client = MagicMock() @@ -118,7 +118,7 @@ class TestInstalledAndLatest: mock_client.return_value.__enter__.return_value = client assert inst._get_latest_version("foo") is None - @patch("app.library.PackageInstaller.httpx.Client") + @patch("app.library.PackageInstaller.sync_client") def test_get_latest_version_exception(self, mock_client, tmp_path: Path) -> None: inst = PackageInstaller(pkg_path=tmp_path) mock_client.side_effect = RuntimeError("boom") diff --git a/sc_short.jpg b/sc_short.jpg index 13c61c35..f20345ec 100644 Binary files a/sc_short.jpg and b/sc_short.jpg differ diff --git a/sc_simple.jpg b/sc_simple.jpg index 6025de32..4d9bc603 100644 Binary files a/sc_simple.jpg and b/sc_simple.jpg differ diff --git a/ui/app/stores/SocketStore.ts b/ui/app/stores/SocketStore.ts index bfc517c4..467147dd 100644 --- a/ui/app/stores/SocketStore.ts +++ b/ui/app/stores/SocketStore.ts @@ -1,4 +1,6 @@ import { io, type Socket as IOSocket, type SocketOptions, type ManagerOptions } from "socket.io-client" +import { ref, readonly } from 'vue' +import { defineStore } from 'pinia' import type { ConfigState } from "~/types/config"; import type { StoreItem } from "~/types/store"; @@ -15,6 +17,8 @@ export const useSocketStore = defineStore('socket', () => { const connectionStatus = ref('disconnected') const error = ref(null) const error_count = ref(0) + const wasHidden = ref(false) + const reconnectTimeout = ref(null) const emit = (event: string, data?: any): any => socket.value?.emit(event, data) const on = (event: string | string[], callback: (...args: any[]) => void, withEvent: boolean = false) => { @@ -33,6 +37,46 @@ export const useSocketStore = defineStore('socket', () => { const getSessionId = (): string | null => socket.value?.id || null + const handleVisibilityChange = () => { + if (document.hidden) { + wasHidden.value = true + return + } + + if (true === wasHidden.value && false === isConnected.value) { + if (null !== reconnectTimeout.value) { + clearTimeout(reconnectTimeout.value) + reconnectTimeout.value = null + } + + reconnectTimeout.value = setTimeout(() => { + if (false === isConnected.value) { + console.debug('[SocketStore] Page visible after background, reconnecting...') + reconnect() + } + reconnectTimeout.value = null + }, 100) + } + + wasHidden.value = false + } + + const setupVisibilityListener = () => { + if (typeof document !== 'undefined') { + document.addEventListener('visibilitychange', handleVisibilityChange) + } + } + + const cleanupVisibilityListener = () => { + if (typeof document !== 'undefined') { + document.removeEventListener('visibilitychange', handleVisibilityChange) + } + if (null !== reconnectTimeout.value) { + clearTimeout(reconnectTimeout.value) + reconnectTimeout.value = null + } + } + const reconnect = () => { if (true === isConnected.value) { return; @@ -49,6 +93,7 @@ export const useSocketStore = defineStore('socket', () => { socket.value = null; isConnected.value = false; connectionStatus.value = 'disconnected'; + cleanupVisibilityListener(); } const connect = () => { @@ -234,6 +279,8 @@ export const useSocketStore = defineStore('socket', () => { on('presets_update', (data: string) => config.update('presets', JSON.parse(data).data || [])); on('dlfields_update', (data: string) => config.update('dl_fields', JSON.parse(data).data || [])); + + setupVisibilityListener(); } if (false === isConnected.value) { @@ -243,7 +290,7 @@ export const useSocketStore = defineStore('socket', () => { return { connect, reconnect, disconnect, on, off, emit, - socket, isConnected, + isConnected, getSessionId, connectionStatus: readonly(connectionStatus) as Readonly>, error: readonly(error) as Readonly>,