danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #531 from arabcoders/dev

Browse source 
Refactor: integrate cf solver for both yt-dlp and httpx
This commit is contained in:
Abdulmohsen 2026-01-02 20:34:48 +03:00 committed by GitHub
commit 67f3a24580
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 1114 additions and 187 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
FAQ.md
View file

@ -575,7 +575,8 @@ This will help in case the premiere has a longer loading screen than usual.
# How to bypass CF challenges?
You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL` environment variable to point to your FlareSolverr instance. For example:
You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL`
environment variable to point to your FlareSolverr instance. For example:
```yaml
services:
@ -602,6 +603,3 @@ services:
```
For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project.
> [!NOTE]
> This will only work for yt-dlp part of the project. Anything else will not be affected by this setting for now.

1
README.md
View file

@ -32,6 +32,7 @@ Example of the Simple mode interface.
* Basic authentication support.
* Supports `curl-cffi`. See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#impersonation). `In docker only`.
* Bundled `pot provider plugin`. See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide). `In docker only`.
* Support using [flaresolverr/flaresolverr](https://github.com/flaresolverr/flaresolverr) to bypass Cloudflare protections for yt-dlp and internal http client. See [related FAQ](FAQ.md#how-to-bypass-cf-challenges).
* Automatic updates for `yt-dlp` and custom `pip` packages. `In docker only`.
* Conditions feature to apply custom options based on `yt-dlp` returned info.
* Custom browser extensions, bookmarklets and iOS shortcuts to send links to YTPTube instance.

16
app/library/DownloadQueue.py
View file

@ -863,8 +863,7 @@ class DownloadQueue(metaclass=Singleton):
if not item.requeued and (condition := Conditions.get_instance().match(info=entry)):
already.pop()
item_title = entry.get("title") or entry.get("id") or item.url
message = f"Condition '{condition.name}' matched for '{item_title}'."
message = f"Condition '{condition.name}' matched for '{item!r}'."
if condition.cli:
message += f" Re-queuing with '{condition.cli}'."
@ -873,18 +872,19 @@ class DownloadQueue(metaclass=Singleton):
if condition.extras.get("ignore_download", False):
extra_msg: str = ""
if yt_conf.get("download_archive") and not condition.extras.get("no_archive", False):
archive_add(yt_conf.get("download_archive"), [archive_id])
extra_msg = f" and added to archive '{yt_conf.get('download_archive')}'"
if _archive_file and not condition.extras.get("no_archive", False):
archive_add(_archive_file, [archive_id])
extra_msg = f" and added to archive file '{_archive_file}'"
log_message = f"Ignoring download of '{item_title}' as per condition '{condition.name}'{extra_msg}."
_name = entry.get("title", entry.get("id"))
log_message = f"Ignoring download of '{_name!r}' as per condition '{condition.name}'{extra_msg}."
store_type, _ = await self.get_item(archive_id=archive_id)
if not store_type:
dlInfo = Download(
info=ItemDTO(
id=entry.get("id"),
title=item_title,
title=_name,
url=item.url,
preset=item.preset,
folder=item.folder,
@ -909,7 +909,7 @@ class DownloadQueue(metaclass=Singleton):
if condition.extras.get("set_preset") and (target_preset := condition.extras.get("set_preset")):
if Presets.get_instance().has(target_preset):
log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item_title}' as per condition '{condition.name}'."
log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item!r}' as per condition '{condition.name}'."
LOG.info(log_message)
self._notify.emit(Events.LOG_INFO, data={}, title="Preset Switched", message=log_message)
item = item.new_with(preset=target_preset)

3
app/library/Notifications.py
View file

@ -15,6 +15,7 @@ from .BackgroundWorker import BackgroundWorker
from .config import Config
from .encoder import Encoder
from .Events import Event, EventBus, Events
from .httpx_client import async_client
from .ItemDTO import Item, ItemDTO
from .Presets import Preset, Presets
from .Singleton import Singleton
@ -147,7 +148,7 @@ class Notification(metaclass=Singleton):
"Debug mode."
self._file: Path = Path(file) if file else Path(config.config_path).joinpath("notifications.json")
"File to store notification targets."
self._client: httpx.AsyncClient = client or httpx.AsyncClient()
self._client: httpx.AsyncClient = client or async_client()
"HTTP client to send requests."
self._encoder: Encoder = encoder or Encoder()
"Encoder to encode data."

6
app/library/PackageInstaller.py
View file

@ -6,7 +6,7 @@ import subprocess
import sys
from pathlib import Path
import httpx
from .httpx_client import sync_client
LOG: logging.Logger = logging.getLogger("package_installer")
@ -17,7 +17,7 @@ def parse_version(v: str) -> tuple[int, ...]:
class Packages:
def __init__(self, env: str | None, file: str | None, upgrade: bool = False):
from_env = env.split() if env else []
from_env: list[str] = env.split() if env else []
from_file = []
if file:
@ -89,7 +89,7 @@ class PackageInstaller:
def _get_latest_version(self, pkg: str) -> str | None:
url: str = f"https://pypi.org/pypi/{pkg}/json"
try:
with httpx.Client(timeout=5.0) as client:
with sync_client(timeout=5.0) as client:
resp = client.get(url)
if 200 == resp.status_code:
return resp.json()["info"]["version"]

234
app/library/CFSolverRH.py → app/library/cf_solver_handler.py
View file

@ -1,15 +1,10 @@
# flake8: noqa: S310
from __future__ import annotations
import http.cookiejar
import json
import logging
import time
import urllib.request
from abc import ABC
from collections.abc import Callable
from typing import Any, ClassVar
from urllib.parse import urlparse
from yt_dlp.networking.common import (
_REQUEST_HANDLERS,
@ -24,15 +19,16 @@ from yt_dlp.networking.common import (
from yt_dlp.networking.exceptions import HTTPError
from yt_dlp.utils.networking import clean_headers
from app.library.cf_solver_shared import CACHE, is_cf_challenge, solver
LOG: logging.Logger = logging.getLogger(__name__)
SolverFn = Callable[[Request, Response, RequestHandler], Request | None]
CacheEntry = dict[str, Any]
def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> Request | None:
"""
A Cloudflare solver that uses FlareSolverr/FlareSolverr to solve challenges.
A Cloudflare solver that uses FlareSolverr to solve challenges.
Args:
request (Request): The original request that triggered the challenge.
@ -43,81 +39,25 @@ def cf_solver(request: Request, _response: Response, handler: RequestHandler) ->
Request | None: The modified request with solved credentials, or None if solving failed.
"""
from app.library.config import Config
config = Config.get_instance()
if not config.flaresolverr_url:
if not isinstance(handler, CFSolverRH):
return None
parsed_endpoint = urlparse(config.flaresolverr_url)
if parsed_endpoint.scheme not in ("http", "https"):
return None
if request.data is not None and request.method not in ("GET", None):
return None
method: str = request.method.lower() if isinstance(request.method, str) else "get"
if method not in ("get", "head"):
method = "get"
payload: dict[str, Any] = {
"cmd": f"request.{method}",
"url": request.url,
"maxTimeout": int(getattr(config, "flaresolverr_max_timeout", 60) * 1000),
}
cookiejar = handler._get_cookiejar(request)
cookies = (
[
{
"name": cookie.name,
"value": cookie.value,
"domain": cookie.domain or urlparse(request.url).hostname or "",
"path": cookie.path or "/",
}
for cookie in cookiejar
]
if cookiejar
else []
)
host: str = handler._get_host(request.url)
cookies = [
{
"name": cookie.name,
"value": cookie.value,
"domain": cookie.domain or host,
"path": cookie.path or "/",
}
for cookie in cookiejar
]
if cookies:
payload["cookies"] = cookies
req = urllib.request.Request(
config.flaresolverr_url,
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
try:
LOG.info(f"Trying to solve Cloudflare challenge for '{request.url}' this may take a while...")
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
result = json.loads(resp.read().decode("utf-8"))
except Exception as e:
LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {e!s}")
solution: dict[str, Any] | None = solver(request.url, cookies, request.headers.get("User-Agent"))
if not handler._apply_solution(solution, request.url, request.headers, cookiejar):
return None
if "ok" != result.get("status"):
LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {result}")
return None
LOG.info(f"Successfully solved Cloudflare challenge for '{request.url}'.")
solution = result.get("solution") or {}
_cookiejar_from_solution(solution.get("cookies"), request, handler)
if ua := solution.get("userAgent"):
request.headers["User-Agent"] = ua
CFSolverRH.cache[urlparse(request.url).netloc] = {
"cookies": solution.get("cookies") or [],
"userAgent": ua,
"expires_at": time.time() + config.flaresolverr_cache_ttl,
}
return request
@ -138,39 +78,6 @@ def set_cf_handler(solver: SolverFn | None = None) -> type[CFSolverRH]:
return CFSolverRH
def _cookiejar_from_solution(cookies, request: Request, handler: RequestHandler) -> None:
cookiejar = handler._get_cookiejar(request)
host = urlparse(request.url).hostname or ""
for cookie in cookies or []:
name = cookie.get("name")
value = cookie.get("value")
if not name or value is None:
continue
domain = cookie.get("domain") or host
path = cookie.get("path") or "/"
cookiejar.set_cookie(
http.cookiejar.Cookie(
version=0,
name=name,
value=value,
port=None,
port_specified=False,
domain=domain,
domain_specified=True,
domain_initial_dot=domain.startswith("."),
path=path,
path_specified=True,
secure=bool(cookie.get("secure")),
expires=cookie.get("expires"),
discard=False,
comment=None,
comment_url=None,
rest={},
rfc2109=False,
)
)
@register_rh
class CFSolverRH(RequestHandler, ABC):
"""Request handler that intercepts Cloudflare challenges"""
@ -178,7 +85,6 @@ class CFSolverRH(RequestHandler, ABC):
_SUPPORTED_URL_SCHEMES = ("http", "https")
_SUPPORTED_PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h")
solver: ClassVar[SolverFn | None] = None
cache: ClassVar[dict[str, CacheEntry]] = {}
def __init__(self, *, solver: SolverFn | None = None, **kwargs) -> None:
super().__init__(**kwargs)
@ -190,12 +96,66 @@ class CFSolverRH(RequestHandler, ABC):
self._fallback_director.close()
self._fallback_director = None
@staticmethod
def _get_host(url: str) -> str:
"""Extract hostname from URL."""
from urllib.parse import urlparse
return urlparse(url).hostname or ""
def _apply_solution(
self,
solution: dict[str, Any] | None,
url: str,
headers: dict[str, Any],
cookiejar: http.cookiejar.CookieJar,
) -> bool:
"""Apply Cloudflare solution (cookies and User-Agent) to request."""
if not solution:
return False
host = self._get_host(url)
for cookie in solution.get("cookies", []):
name = cookie.get("name")
value = cookie.get("value")
if not name or value is None:
continue
domain = cookie.get("domain") or host
path = cookie.get("path") or "/"
cookiejar.set_cookie(
http.cookiejar.Cookie(
version=0,
name=name,
value=value,
port=None,
port_specified=False,
domain=domain,
domain_specified=True,
domain_initial_dot=domain.startswith("."),
path=path,
path_specified=True,
secure=bool(cookie.get("secure")),
expires=cookie.get("expires"),
discard=False,
comment=None,
comment_url=None,
rest={},
rfc2109=False,
)
)
if ua := solution.get("userAgent"):
headers["User-Agent"] = ua
return True
def _check_extensions(self, extensions) -> None:
super()._check_extensions(extensions)
for key in ("cookiejar", "timeout", "legacy_ssl", "keep_header_casing", "cf_retry"):
extensions.pop(key, None)
def _validate(self, request: Request): # type: ignore[override]
def _validate(self, request: Request):
self._check_url_scheme(request)
self._check_proxies(request.proxies or self.proxies)
extensions = request.extensions.copy()
@ -230,30 +190,6 @@ class CFSolverRH(RequestHandler, ABC):
self._fallback_director = director
return director
@staticmethod
def _is_cf_response(response: Response) -> bool:
"""
Check if the response is a Cloudflare challenge response.
Args:
response (Response): The HTTP response to check.
Returns:
bool: True if the response is a Cloudflare challenge, False otherwise.
"""
status: int | None = getattr(response, "status", None)
if status not in (403, 429, 503):
return False
headers = response.headers or {}
server_header: str = (headers.get("Server") or "").lower()
if "cloudflare" in server_header:
return True
cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor")
return any(key in headers for key in cf_header_keys)
def _solve(self, request: Request, response: Response) -> Request | None:
return self._solver(request, response, self) if self._solver else None
@ -276,42 +212,32 @@ class CFSolverRH(RequestHandler, ABC):
response.close()
return director.send(solved_request)
def _inject_cookie(self, request: Request) -> None:
cache_key: str = urlparse(request.url).netloc
if not (cached := self.cache.get(cache_key)):
return
if cached.get("expires_at", 0) <= time.time():
self.cache.pop(cache_key, None)
return
LOG.info(f"Injecting cached Cloudflare cookies for '{cache_key}'.")
_cookiejar_from_solution(cached.get("cookies"), request, self)
if ua := cached.get("userAgent"):
request.headers["User-Agent"] = ua
def _send(self, request: Request) -> Response:
self._inject_cookie(request)
host: str = self._get_host(request.url)
if host and (cached := CACHE.get(host)):
LOG.info(f"Injecting cached Cloudflare cookies for '{host}'.")
self._apply_solution(cached, request.url, request.headers, self._get_cookiejar(request))
director: RequestDirector = self._build_fallback()
try:
response: Response = director.send(request)
except HTTPError as error:
if error.response and self._is_cf_response(error.response):
if error.response and is_cf_challenge(getattr(error.response, "status", None), error.response.headers):
return self._retry_with_clearance(request, error.response, director)
raise
if self._is_cf_response(response):
if is_cf_challenge(getattr(response, "status", None), response.headers):
return self._retry_with_clearance(request, response, director)
return response
@register_preference(CFSolverRH)
def cf_solver_preference(_handler: RequestHandler, _request: Request) -> int:
def cf_solver_preference(_handler, _request) -> int:
from app.library.config import Config
if not Config.get_instance().flaresolverr_url:
return 0
return 100
return 500

100
app/library/cf_solver_shared.py Normal file
View file

@ -0,0 +1,100 @@
# flake8: noqa: S310
from __future__ import annotations
import json
import logging
import time
import urllib.request
from typing import Any
from urllib.parse import urlparse
from .cache import Cache
CACHE: Cache = Cache()
LOG: logging.Logger = logging.getLogger("cf_solver")
def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> dict[str, Any] | None:
"""
Run FlareSolverr solve. Returns solution dict or None.
Args:
url (str): The URL to solve the challenge for.
cookies (list[dict]): List of existing cookies to send to FlareSolverr.
user_agent (str | None): The User-Agent string to send to FlareSolverr.
Returns:
dict[str, Any] | None: The solution dict from FlareSolverr, or None if solving fails.
"""
from app.library.config import Config
config = Config.get_instance()
if not (endpoint := config.flaresolverr_url):
return None
parsed = urlparse(endpoint)
if parsed.scheme not in ("http", "https"):
return None
host = urlparse(url).hostname or ""
if not host:
return None
if cached := CACHE.get(host):
return cached
payload: dict[str, Any] = {
"cmd": "request.get",
"url": url,
"maxTimeout": int(config.flaresolverr_max_timeout * 1000),
}
if cookies:
payload["cookies"] = cookies
if user_agent:
payload.setdefault("headers", {})["User-Agent"] = user_agent
req = urllib.request.Request(
endpoint,
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
LOG.info(f"Solving Cloudflare challenge for '{host}' via FlareSolverr.")
start_time = time.time()
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
result = json.loads(resp.read().decode("utf-8"))
if "ok" != result.get("status"):
LOG.error(f"FlareSolverr failed to solve challenge for '{host}': {result.get('message')}")
return None
LOG.info(f"FlareSolverr successfully solved challenge for '{host}'. (Took {time.time() - start_time:.2f} seconds)")
solution = result.get("solution") or {}
CACHE.set(
host,
{"cookies": solution.get("cookies") or [], "userAgent": solution.get("userAgent")},
ttl=config.flaresolverr_cache_ttl,
)
return CACHE.get(host)
def is_cf_challenge(status: int | None, headers: dict[str, Any] | None) -> bool:
"""
Determine whether a response indicates a Cloudflare challenge.
"""
if status not in (403, 429, 503):
return False
headers = headers or {}
server_header: str = str(headers.get("Server", "")).lower()
if "cloudflare" in server_header:
return True
cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor")
return any(key in headers for key in cf_header_keys)

154
app/library/httpx_client.py Normal file
View file

@ -0,0 +1,154 @@
from __future__ import annotations
import logging
from typing import Any
import httpx
from .cf_solver_shared import is_cf_challenge, solver
__all__: list[str] = ["async_client", "sync_client"]
LOG: logging.Logger = logging.getLogger("httpx_cf")
def _parse_cookie_header(cookie_header: str | None) -> dict[str, str]:
cookies: dict[str, str] = {}
if not cookie_header:
return cookies
for item in cookie_header.split(";"):
item: str = item.strip()
if "=" in item:
name, value = item.split("=", 1)
cookies[name] = value
return cookies
def _merge_cookies(existing_header: str | None, new_cookies: list[dict[str, str]]) -> dict[str, str]:
merged: dict[str, str] = _parse_cookie_header(existing_header)
for cookie in new_cookies:
if cookie.get("name") and cookie.get("value"):
merged[cookie["name"]] = cookie["value"]
return merged
def _get_transport(
enable_cf: bool,
is_async: bool,
transport: httpx.AsyncBaseTransport | None,
) -> httpx.AsyncBaseTransport:
if enable_cf:
return CFAsyncTransport(base=transport) if is_async else CFTransport(base=transport)
return transport or (httpx.AsyncHTTPTransport() if is_async else httpx.HTTPTransport())
class CFAsyncTransport(httpx.AsyncBaseTransport):
def __init__(self, base: httpx.AsyncBaseTransport | None = None):
self.base: httpx.AsyncBaseTransport | httpx.AsyncHTTPTransport = base or httpx.AsyncHTTPTransport()
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
response: httpx.Response = await self.base.handle_async_request(request)
if not is_cf_challenge(response.status_code, dict(response.headers)):
return response
url = str(request.url)
solution: dict[str, Any] | None = solver(url, [], request.headers.get("User-Agent"))
if not solution:
return response
if cookies := solution.get("cookies", []):
merged_cookies: dict[str, str] = _merge_cookies(request.headers.get("Cookie"), cookies)
headers: httpx.Headers = request.headers.copy()
headers.pop("Cookie", None)
request = httpx.Request(
method=request.method,
url=request.url,
headers=headers,
content=request.content,
cookies=merged_cookies,
)
if ua := solution.get("userAgent"):
request.headers["User-Agent"] = ua
await response.aclose()
return await self.base.handle_async_request(request)
def close(self) -> None:
self.base.close()
class CFTransport(httpx.BaseTransport):
def __init__(self, base: httpx.BaseTransport | None = None):
self.base: httpx.BaseTransport | httpx.HTTPTransport = base or httpx.HTTPTransport()
def handle_request(self, request: httpx.Request) -> httpx.Response:
response: httpx.Response = self.base.handle_request(request)
if not is_cf_challenge(response.status_code, dict(response.headers)):
return response
url = str(request.url)
solution: dict[str, Any] | None = solver(url, [], request.headers.get("User-Agent"))
if not solution:
return response
if cookies := solution.get("cookies", []):
merged_cookies: dict[str, str] = _merge_cookies(request.headers.get("Cookie"), cookies)
headers: httpx.Headers = request.headers.copy()
headers.pop("Cookie", None)
request = httpx.Request(
method=request.method,
url=request.url,
headers=headers,
content=request.content,
cookies=merged_cookies,
)
if ua := solution.get("userAgent"):
request.headers["User-Agent"] = ua
response.close()
return self.base.handle_request(request)
def close(self) -> None:
self.base.close()
def async_client(enable_cf: bool = True, **kwargs: Any) -> httpx.AsyncClient:
"""
Create an httpx.AsyncClient with optional Cloudflare challenge solving.
Args:
enable_cf (bool): Whether to enable Cloudflare challenge solving.
**kwargs: Additional keyword arguments to pass to httpx.AsyncClient.
Returns:
httpx.AsyncClient: The configured httpx.AsyncClient instance.
"""
transport = kwargs.pop("transport", None)
return httpx.AsyncClient(transport=_get_transport(enable_cf, is_async=True, transport=transport), **kwargs)
def sync_client(enable_cf: bool = True, **kwargs: Any) -> httpx.Client:
"""
Create an httpx.Client with optional Cloudflare challenge solving.
Args:
enable_cf (bool): Whether to enable Cloudflare challenge solving.
**kwargs: Additional keyword arguments to pass to httpx.Client.
Returns:
httpx.Client: The configured httpx.Client instance.
"""
transport = kwargs.pop("transport", None)
return httpx.Client(transport=_get_transport(enable_cf, is_async=False, transport=transport), **kwargs)

3
app/library/task_handlers/_base_handler.py
View file

@ -5,6 +5,7 @@ import httpx
from yt_dlp.utils.networking import random_user_agent
from app.library.config import Config
from app.library.httpx_client import async_client
from app.library.Tasks import Task, TaskFailure, TaskResult
@ -73,7 +74,7 @@ class BaseHandler:
if proxy := ytdlp_opts.get("proxy", None):
opts["proxy"] = proxy
async with httpx.AsyncClient(**opts) as client:
async with async_client(**opts) as client:
method = kwargs.pop("method", "GET").upper()
timeout = ytdlp_opts.get("timeout", ytdlp_opts.get("socket_timeout", 120))
return await client.request(method=method, url=url, timeout=timeout, **kwargs)

7
app/library/task_handlers/generic.py
View file

@ -14,7 +14,6 @@ from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal
from urllib.parse import urljoin
import httpx
import jmespath
from parsel import Selector
from parsel.selector import SelectorList
@ -22,12 +21,14 @@ from yt_dlp.utils.networking import random_user_agent
from app.library.cache import Cache
from app.library.config import Config
from app.library.httpx_client import async_client
from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult
from app.library.Utils import extract_info, get_archive_id
from ._base_handler import BaseHandler
if TYPE_CHECKING:
import httpx
from parsel.selector import SelectorList
LOG: logging.Logger = logging.getLogger(__name__)
@ -474,7 +475,7 @@ def load_task_definitions(config: Config | None = None) -> list[TaskDefinition]:
LOG.error(f"[{path.name}] Unsupported response type '{response_type}'.")
continue
response_config = ResponseConfig(format=response_type) # type: ignore[arg-type]
response_config = ResponseConfig(format=response_type)
parse_raw: Mapping | None = raw.get("parse")
if not isinstance(parse_raw, Mapping):
@ -826,7 +827,7 @@ class GenericTaskHandler(BaseHandler):
timeout_value: float | Any = definition.request.timeout or ytdlp_opts.get("socket_timeout", 120)
async with httpx.AsyncClient(**client_options) as client:
async with async_client(**client_options) as client:
response: httpx.Response = await client.request(
method=definition.request.normalized_method(),
url=url,

4
app/library/ytdlp.py
View file

@ -1,11 +1,11 @@
# flake8: noqa: F401
import sys
from typing import Any
import yt_dlp
from yt_dlp.utils import make_archive_id
# imported to register the handler with yt-dlp.
from app.library.CFSolverRH import set_cf_handler
from app.library.cf_solver_handler import set_cf_handler
class _ArchiveProxy:

5
app/routes/api/docs.py
View file

@ -3,7 +3,6 @@ import time
from datetime import UTC, datetime
from typing import Any
import httpx
from aiohttp import web
from aiohttp.web import Request, Response
from yt_dlp.utils.networking import random_user_agent
@ -86,7 +85,9 @@ async def get_doc(request: Request, config: Config, cache: Cache) -> Response:
except Exception:
pass
async with httpx.AsyncClient(**opts) as client:
from app.library.httpx_client import async_client
async with async_client(**opts) as client:
LOG.debug(f"Fetching doc from '{url}'.")
response = await client.request(method="GET", url=url, follow_redirects=True)
dct = {

9
app/routes/api/images.py
View file

@ -5,7 +5,6 @@ from datetime import UTC, datetime
from typing import Any
from urllib.parse import urlparse
import httpx
from aiohttp import web
from aiohttp.web import Request, Response
from yt_dlp.utils.networking import random_user_agent
@ -67,7 +66,9 @@ async def get_thumbnail(request: Request, config: Config) -> Response:
except Exception:
pass
async with httpx.AsyncClient(**opts) as client:
from app.library.httpx_client import async_client
async with async_client(**opts) as client:
LOG.debug(f"Fetching thumbnail from '{url}'.")
response = await client.request(method="GET", url=url, follow_redirects=True)
@ -158,7 +159,9 @@ async def get_background(request: Request, config: Config, cache: Cache) -> Resp
except Exception:
pass
async with httpx.AsyncClient(**opts) as client:
from app.library.httpx_client import async_client
async with async_client(**opts) as client:
if backend.startswith("https://www.bing.com/HPImageArchive.aspx"):
if not cache.has(CACHE_KEY_BING):
response = await client.request(method="GET", url=backend)

5
app/routes/api/tasks.py
View file

@ -307,9 +307,10 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R
filename.write_text(xml_content, encoding="utf-8")
try:
import httpx
from yt_dlp.utils.networking import random_user_agent
from app.library.httpx_client import async_client
ytdlp_args: dict = task.get_ytdlp_opts().get_all()
opts: dict[str, Any] = {
"headers": {
@ -331,7 +332,7 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R
except Exception:
pass
async with httpx.AsyncClient(**opts) as client:
async with async_client(**opts) as client:
for key in info.get("thumbnails", {}):
try:
url = info["thumbnails"][key]

257
app/tests/test_cf_solver_handler.py Normal file
View file

@ -0,0 +1,257 @@
"""Comprehensive tests for cf_solver_handler yt-dlp request handler."""
from __future__ import annotations
import http.cookiejar
from unittest.mock import Mock, patch
import pytest
try:
from yt_dlp.networking.common import Request, RequestDirector, Response
from yt_dlp.networking.exceptions import HTTPError
YTDLP_AVAILABLE = True
except ImportError:
YTDLP_AVAILABLE = False
Request = Mock
Response = Mock
RequestDirector = Mock
HTTPError = Exception
pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available")
@pytest.fixture(scope="module")
def cf_handler_module():
"""Lazily import cf_solver_handler module to avoid circular imports."""
from app.library import cf_solver_handler
return cf_solver_handler
class TestCfSolverFunction:
"""Test the cf_solver function."""
@patch("app.library.cf_solver_handler.solver")
def test_cf_solver_success(self, mock_solver, cf_handler_module):
"""Test successful solving of Cloudflare challenge."""
mock_solver.return_value = {"cookies": [], "userAgent": "Mozilla/5.0"}
handler = cf_handler_module.CFSolverRH(logger=Mock())
cookiejar = http.cookiejar.CookieJar()
handler._get_cookiejar = Mock(return_value=cookiejar)
request = Mock()
request.url = "https://example.com/path"
request.headers = {"User-Agent": "test"}
response = Mock()
result = cf_handler_module.cf_solver(request, response, handler)
assert result is request
mock_solver.assert_called_once()
@patch("app.library.cf_solver_handler.solver")
def test_cf_solver_no_solution(self, mock_solver, cf_handler_module):
"""Test when solver returns no solution."""
mock_solver.return_value = None
handler = cf_handler_module.CFSolverRH(logger=Mock())
cookiejar = http.cookiejar.CookieJar()
handler._get_cookiejar = Mock(return_value=cookiejar)
request = Mock()
request.url = "https://example.com/path"
request.headers = {"User-Agent": "test"}
response = Mock()
result = cf_handler_module.cf_solver(request, response, handler)
assert result is None
mock_solver.assert_called_once()
@patch("app.library.cf_solver_handler.solver")
def test_cf_solver_with_existing_cookies(self, mock_solver, cf_handler_module):
"""Test solving with existing cookies in jar."""
mock_solver.return_value = {"cookies": [], "userAgent": "Mozilla/5.0"}
handler = cf_handler_module.CFSolverRH(logger=Mock())
cookiejar = http.cookiejar.CookieJar()
cookie = http.cookiejar.Cookie(
version=0,
name="existing",
value="value",
port=None,
port_specified=False,
domain="example.com",
domain_specified=True,
domain_initial_dot=False,
path="/",
path_specified=True,
secure=False,
expires=None,
discard=True,
comment=None,
comment_url=None,
rest={},
rfc2109=False,
)
cookiejar.set_cookie(cookie)
handler._get_cookiejar = Mock(return_value=cookiejar)
request = Mock()
request.url = "https://example.com/path"
request.headers = {"User-Agent": "test"}
response = Mock()
result = cf_handler_module.cf_solver(request, response, handler)
assert result is request
call_args = mock_solver.call_args
assert call_args is not None
cookies_arg = call_args[0][1]
assert len(cookies_arg) > 0
assert "existing" == cookies_arg[0]["name"]
class TestSetCfHandler:
"""Test set_cf_handler function."""
def test_set_cf_handler_default(self, cf_handler_module):
"""Test setting CF handler with default solver."""
result = cf_handler_module.set_cf_handler()
assert result is cf_handler_module.CFSolverRH
assert cf_handler_module.CFSolverRH.solver is None or callable(cf_handler_module.CFSolverRH.solver)
def test_set_cf_handler_custom_solver(self, cf_handler_module):
"""Test setting CF handler with custom solver."""
def custom_solver(req, resp, handler):
return req
result = cf_handler_module.set_cf_handler(custom_solver)
assert result is cf_handler_module.CFSolverRH
assert cf_handler_module.CFSolverRH.solver is custom_solver
class TestCFSolverRH:
"""Test CFSolverRH request handler class."""
@pytest.fixture(autouse=True)
def setup(self, cf_handler_module):
"""Set up test fixtures."""
self.module = cf_handler_module
self.handler = self.module.CFSolverRH(logger=Mock(), verbose=False)
def test_init_default(self):
"""Test initialization with defaults."""
handler = self.module.CFSolverRH(logger=Mock(), verbose=False)
assert handler._solver is not None
assert handler._fallback_director is None
def test_init_custom_solver(self):
"""Test initialization with custom solver."""
def custom_solver(req, resp, handler):
return req
handler = self.module.CFSolverRH(logger=Mock(), verbose=False, solver=custom_solver)
assert handler._solver is custom_solver
def test_close(self):
"""Test closing handler."""
mock_director = Mock()
self.handler._fallback_director = mock_director
self.handler.close()
mock_director.close.assert_called_once()
assert self.handler._fallback_director is None
def test_close_no_director(self):
"""Test closing handler when no director exists."""
self.handler._fallback_director = None
self.handler.close()
def test_check_extensions(self):
"""Test extension checking."""
extensions = {
"timeout": 30,
"legacy_ssl": True,
"other": "value",
}
self.handler._check_extensions(extensions)
def test_validate(self):
"""Test request validation."""
request = Mock()
request.url = "https://example.com"
request.proxies = None
request.extensions = {}
self.handler._validate(request)
def test_solve(self):
"""Test solving challenge."""
request = Mock()
request.url = "https://example.com"
request.headers = {}
request.extensions = {}
response = Mock()
self.handler._solver = Mock(return_value=request)
result = self.handler._solve(request, response)
assert result is request
self.handler._solver.assert_called_once()
self.handler._solver = None
result = self.handler._solve(request, response)
assert result is None
def test_mark_retry(self):
"""Test marking request as retry."""
request = Mock()
request.copy = Mock(return_value=Mock())
request.copy.return_value.extensions = {}
new_request = self.module.CFSolverRH._mark_retry(request)
assert new_request.extensions.get("cf_retry") is True
class TestCfSolverPreference:
"""Test cf_solver_preference function."""
def test_preference_with_flaresolverr(self, cf_handler_module, monkeypatch):
"""Test preference when FlareSolverr is configured."""
mock_config = Mock()
mock_config.flaresolverr_url = "http://localhost:8191/v1"
import app.library.config
monkeypatch.setattr(app.library.config.Config, "get_instance", lambda: mock_config)
def test_preference_without_flaresolverr(self, cf_handler_module, monkeypatch):
"""Test preference when FlareSolverr is not configured."""
mock_config = Mock()
mock_config.flaresolverr_url = None
import app.library.config
monkeypatch.setattr(app.library.config.Config, "get_instance", lambda: mock_config)
def test_preference_with_empty_flaresolverr(self, cf_handler_module, monkeypatch):
"""Test preference when FlareSolverr URL is empty."""
mock_config = Mock()
mock_config.flaresolverr_url = ""
import app.library.config
monkeypatch.setattr(app.library.config.Config, "get_instance", lambda: mock_config)

436
app/tests/test_httpx_client.py Normal file
View file

@ -0,0 +1,436 @@
"""Comprehensive tests for httpx_client Cloudflare challenge solving."""
from __future__ import annotations
import http.cookiejar
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import httpx
import pytest
from app.library.httpx_client import (
CFAsyncTransport,
CFTransport,
_get_transport,
async_client,
sync_client,
)
class TestGetTransport:
"""Test transport factory function."""
def test_get_transport_cf_enabled_async(self):
"""Test getting async transport with CF enabled."""
transport = _get_transport(enable_cf=True, is_async=True, transport=None)
assert isinstance(transport, CFAsyncTransport)
def test_get_transport_cf_enabled_sync(self):
"""Test getting sync transport with CF enabled."""
transport = _get_transport(enable_cf=False, is_async=False, transport=None)
assert isinstance(transport, httpx.HTTPTransport)
def test_get_transport_cf_disabled_async(self):
"""Test getting async transport with CF disabled."""
transport = _get_transport(enable_cf=False, is_async=True, transport=None)
assert isinstance(transport, httpx.AsyncHTTPTransport)
def test_get_transport_cf_disabled_sync(self):
"""Test getting sync transport with CF disabled."""
transport = _get_transport(enable_cf=False, is_async=False, transport=None)
assert isinstance(transport, httpx.HTTPTransport)
def test_get_transport_custom_base_async(self):
"""Test getting transport with custom base transport."""
custom_transport = httpx.AsyncHTTPTransport()
transport = _get_transport(enable_cf=True, is_async=True, transport=custom_transport)
assert isinstance(transport, CFAsyncTransport)
assert transport.base is custom_transport
def test_get_transport_custom_base_sync(self):
"""Test getting transport with custom base transport."""
custom_transport = httpx.HTTPTransport()
transport = _get_transport(enable_cf=True, is_async=False, transport=custom_transport)
assert isinstance(transport, CFTransport)
assert transport.base is custom_transport
class TestCFAsyncTransport:
"""Test async Cloudflare transport wrapper."""
def setup_method(self):
"""Set up test fixtures."""
self.base_transport = AsyncMock(spec=httpx.AsyncHTTPTransport)
self.transport = CFAsyncTransport(base=self.base_transport)
@pytest.mark.asyncio
async def test_init_default(self):
"""Test initialization with defaults."""
transport = CFAsyncTransport()
assert isinstance(transport.base, httpx.AsyncHTTPTransport)
@pytest.mark.asyncio
async def test_init_custom(self):
"""Test initialization with custom base."""
assert self.transport.base is self.base_transport
@pytest.mark.asyncio
@patch("app.library.httpx_client.is_cf_challenge")
async def test_handle_request_no_challenge(self, mock_is_cf):
"""Test handling request without Cloudflare challenge."""
mock_is_cf.return_value = False
request = httpx.Request("GET", "https://example.com")
response = Mock(status_code=200, headers={})
self.base_transport.handle_async_request.return_value = response
result = await self.transport.handle_async_request(request)
assert result is response
self.base_transport.handle_async_request.assert_called_once()
mock_is_cf.assert_called_once_with(200, {})
@pytest.mark.asyncio
@pytest.mark.asyncio
@patch("app.library.httpx_client.solver")
@patch("app.library.httpx_client.is_cf_challenge")
async def test_handle_request_cached_solution(self, mock_is_cf, mock_solver):
"""Test handling request with Cloudflare solution from cache."""
mock_is_cf.return_value = True
mock_solver.return_value = {
"cookies": [{"name": "cf_clearance", "value": "token123"}],
"userAgent": "Mozilla/5.0",
}
request = httpx.Request("GET", "https://example.com")
response1 = Mock(status_code=403, headers={"Server": "cloudflare"})
response1.aclose = AsyncMock()
response2 = Mock(status_code=200, headers={})
self.base_transport.handle_async_request.side_effect = [response1, response2]
result = await self.transport.handle_async_request(request)
assert result is response2
assert 2 == self.base_transport.handle_async_request.call_count
mock_solver.assert_called_once()
response1.aclose.assert_called_once()
second_call_request = self.base_transport.handle_async_request.call_args_list[1][0][0]
assert "cf_clearance=token123" == second_call_request.headers["Cookie"]
assert "Mozilla/5.0" == second_call_request.headers["User-Agent"]
@pytest.mark.asyncio
@patch("app.library.httpx_client.solver")
@patch("app.library.httpx_client.is_cf_challenge")
async def test_handle_request_solve_challenge(self, mock_is_cf, mock_solver):
"""Test handling request by solving Cloudflare challenge."""
mock_is_cf.return_value = True
mock_solver.return_value = {
"cookies": [{"name": "cf_clearance", "value": "token123"}, {"name": "__cf_bm", "value": "bm456"}],
"userAgent": "Mozilla/5.0",
}
request = httpx.Request("GET", "https://example.com")
response1 = Mock(status_code=403, headers={"cf-ray": "123"})
response1.aclose = AsyncMock()
response2 = Mock(status_code=200, headers={})
self.base_transport.handle_async_request.side_effect = [response1, response2]
result = await self.transport.handle_async_request(request)
assert result is response2
assert 2 == self.base_transport.handle_async_request.call_count
mock_solver.assert_called_once()
response1.aclose.assert_called_once()
second_call_request = self.base_transport.handle_async_request.call_args_list[1][0][0]
assert "cf_clearance=token123; __cf_bm=bm456" == second_call_request.headers["Cookie"]
assert "Mozilla/5.0" == second_call_request.headers["User-Agent"]
@pytest.mark.asyncio
@patch("app.library.httpx_client.solver")
@patch("app.library.httpx_client.is_cf_challenge")
async def test_handle_request_merge_existing_cookies(self, mock_is_cf, mock_solver):
"""Test that existing cookies are preserved when adding CF cookies."""
mock_is_cf.return_value = True
mock_solver.return_value = {
"cookies": [{"name": "cf_clearance", "value": "token123"}],
"userAgent": "Mozilla/5.0",
}
request = httpx.Request("GET", "https://example.com", cookies={"session_id": "abc123", "user_pref": "dark"})
response1 = Mock(status_code=403, headers={"cf-ray": "123"})
response1.aclose = AsyncMock()
response2 = Mock(status_code=200, headers={})
self.base_transport.handle_async_request.side_effect = [response1, response2]
result = await self.transport.handle_async_request(request)
assert result is response2
second_call_request = self.base_transport.handle_async_request.call_args_list[1][0][0]
cookie_header = second_call_request.headers["Cookie"]
assert "session_id=abc123" in cookie_header
assert "user_pref=dark" in cookie_header
assert "cf_clearance=token123" in cookie_header
@pytest.mark.asyncio
@patch("app.library.httpx_client.solver")
@patch("app.library.httpx_client.is_cf_challenge")
async def test_handle_request_solve_failed(self, mock_is_cf, mock_solver):
"""Test handling request when solving fails."""
mock_is_cf.return_value = True
mock_solver.return_value = None
request = httpx.Request("GET", "https://example.com")
response = Mock(status_code=403, headers={"cf-ray": "123"})
response.close = Mock()
self.base_transport.handle_async_request.return_value = response
result = await self.transport.handle_async_request(request)
assert result is response
assert 1 == self.base_transport.handle_async_request.call_count
mock_solver.assert_called_once()
response.close.assert_not_called()
def test_close(self):
"""Test closing transport."""
self.base_transport.close = Mock()
self.transport.close()
self.base_transport.close.assert_called_once()
class TestCFTransport:
"""Test sync Cloudflare transport wrapper."""
def setup_method(self):
"""Set up test fixtures."""
self.base_transport = Mock(spec=httpx.HTTPTransport)
self.transport = CFTransport(base=self.base_transport)
def test_init_default(self):
"""Test initialization with defaults."""
transport = CFTransport()
assert isinstance(transport.base, httpx.HTTPTransport)
def test_init_custom(self):
"""Test initialization with custom base."""
assert self.transport.base is self.base_transport
@patch("app.library.httpx_client.is_cf_challenge")
def test_handle_request_no_challenge(self, mock_is_cf):
"""Test handling request without Cloudflare challenge."""
mock_is_cf.return_value = False
request = httpx.Request("GET", "https://example.com")
response = Mock(status_code=200, headers={})
self.base_transport.handle_request.return_value = response
result = self.transport.handle_request(request)
assert result is response
self.base_transport.handle_request.assert_called_once()
mock_is_cf.assert_called_once_with(200, {})
@patch("app.library.httpx_client.solver")
@patch("app.library.httpx_client.is_cf_challenge")
def test_handle_request_cached_solution(self, mock_is_cf, mock_solver):
"""Test handling request with Cloudflare solution from cache."""
mock_is_cf.return_value = True
mock_solver.return_value = {
"cookies": [{"name": "cf_clearance", "value": "token123"}],
"userAgent": "Mozilla/5.0",
}
request = httpx.Request("GET", "https://example.com")
response1 = Mock(status_code=403, headers={"Server": "cloudflare"})
response1.close = Mock()
response2 = Mock(status_code=200, headers={})
self.base_transport.handle_request.side_effect = [response1, response2]
result = self.transport.handle_request(request)
assert result is response2
assert 2 == self.base_transport.handle_request.call_count
mock_solver.assert_called_once()
response1.close.assert_called_once()
second_call_request = self.base_transport.handle_request.call_args_list[1][0][0]
assert "cf_clearance=token123" == second_call_request.headers["Cookie"]
assert "Mozilla/5.0" == second_call_request.headers["User-Agent"]
@patch("app.library.httpx_client.solver")
@patch("app.library.httpx_client.is_cf_challenge")
def test_handle_request_solve_challenge(self, mock_is_cf, mock_solver):
"""Test handling request by solving Cloudflare challenge."""
mock_is_cf.return_value = True
mock_solver.return_value = {
"cookies": [{"name": "cf_clearance", "value": "token123"}, {"name": "__cf_bm", "value": "bm456"}],
"userAgent": "Mozilla/5.0",
}
request = httpx.Request("GET", "https://example.com")
response1 = Mock(status_code=503, headers={"cf-cache-status": "DYNAMIC"})
response1.close = Mock()
response2 = Mock(status_code=200, headers={})
self.base_transport.handle_request.side_effect = [response1, response2]
result = self.transport.handle_request(request)
assert result is response2
assert 2 == self.base_transport.handle_request.call_count
mock_solver.assert_called_once()
response1.close.assert_called_once()
second_call_request = self.base_transport.handle_request.call_args_list[1][0][0]
assert "cf_clearance=token123; __cf_bm=bm456" == second_call_request.headers["Cookie"]
assert "Mozilla/5.0" == second_call_request.headers["User-Agent"]
@patch("app.library.httpx_client.solver")
@patch("app.library.httpx_client.is_cf_challenge")
def test_handle_request_merge_existing_cookies(self, mock_is_cf, mock_solver):
"""Test that existing cookies are preserved when adding CF cookies."""
mock_is_cf.return_value = True
mock_solver.return_value = {
"cookies": [{"name": "cf_clearance", "value": "token123"}],
"userAgent": "Mozilla/5.0",
}
request = httpx.Request("GET", "https://example.com", cookies={"session_id": "abc123", "user_pref": "dark"})
response1 = Mock(status_code=403, headers={"cf-ray": "123"})
response1.close = Mock()
response2 = Mock(status_code=200, headers={})
self.base_transport.handle_request.side_effect = [response1, response2]
result = self.transport.handle_request(request)
assert result is response2
second_call_request = self.base_transport.handle_request.call_args_list[1][0][0]
cookie_header = second_call_request.headers["Cookie"]
assert "session_id=abc123" in cookie_header
assert "user_pref=dark" in cookie_header
assert "cf_clearance=token123" in cookie_header
@patch("app.library.httpx_client.solver")
@patch("app.library.httpx_client.is_cf_challenge")
def test_handle_request_solve_failed(self, mock_is_cf, mock_solver):
"""Test handling request when solving fails."""
mock_is_cf.return_value = True
mock_solver.return_value = None
request = httpx.Request("GET", "https://example.com")
response = Mock(status_code=429, headers={"cf-ray": "abc123"})
response.close = Mock()
self.base_transport.handle_request.return_value = response
result = self.transport.handle_request(request)
assert result is response
assert 1 == self.base_transport.handle_request.call_count
mock_solver.assert_called_once()
response.close.assert_not_called()
def test_close(self):
"""Test closing transport."""
self.base_transport.close = Mock()
self.transport.close()
self.base_transport.close.assert_called_once()
class TestAsyncClient:
"""Test async client factory function."""
@pytest.mark.asyncio
async def test_async_client_cf_enabled(self):
"""Test creating async client with CF enabled."""
async with async_client(enable_cf=True) as client:
assert isinstance(client, httpx.AsyncClient)
assert isinstance(client._transport, CFAsyncTransport)
@pytest.mark.asyncio
async def test_async_client_cf_disabled(self):
"""Test creating async client with CF disabled."""
async with async_client(enable_cf=False) as client:
assert isinstance(client, httpx.AsyncClient)
assert isinstance(client._transport, httpx.AsyncHTTPTransport)
@pytest.mark.asyncio
async def test_async_client_kwargs(self):
"""Test creating async client with additional kwargs."""
async with async_client(enable_cf=True, timeout=30.0, follow_redirects=True) as client:
assert isinstance(client, httpx.AsyncClient)
assert isinstance(client._transport, CFAsyncTransport)
assert 30.0 == client.timeout.read
assert client.follow_redirects is True
@pytest.mark.asyncio
async def test_async_client_custom_transport(self):
"""Test creating async client with custom transport."""
custom = httpx.AsyncHTTPTransport()
async with async_client(enable_cf=True, transport=custom) as client:
assert isinstance(client, httpx.AsyncClient)
assert isinstance(client._transport, CFAsyncTransport)
assert client._transport.base is custom
@pytest.mark.asyncio
async def test_async_client_custom_transport_cf_disabled(self):
"""Test creating async client with custom transport and CF disabled."""
custom = httpx.AsyncHTTPTransport()
async with async_client(enable_cf=False, transport=custom) as client:
assert isinstance(client, httpx.AsyncClient)
assert client._transport is custom
class TestSyncClient:
"""Test sync client factory function."""
def test_sync_client_cf_enabled(self):
"""Test creating sync client with CF enabled."""
client = sync_client(enable_cf=True)
assert isinstance(client, httpx.Client)
assert isinstance(client._transport, CFTransport)
client.close()
def test_sync_client_cf_disabled(self):
"""Test creating sync client with CF disabled."""
client = sync_client(enable_cf=False)
assert isinstance(client, httpx.Client)
assert isinstance(client._transport, httpx.HTTPTransport)
client.close()
def test_sync_client_kwargs(self):
"""Test creating sync client with additional kwargs."""
client = sync_client(enable_cf=True, timeout=30.0, follow_redirects=True)
assert isinstance(client, httpx.Client)
assert isinstance(client._transport, CFTransport)
assert 30.0 == client.timeout.read
assert client.follow_redirects is True
client.close()
def test_sync_client_custom_transport(self):
"""Test creating sync client with custom transport."""
custom = httpx.HTTPTransport()
client = sync_client(enable_cf=True, transport=custom)
assert isinstance(client, httpx.Client)
assert isinstance(client._transport, CFTransport)
assert client._transport.base is custom
client.close()
def test_sync_client_custom_transport_cf_disabled(self):
"""Test creating sync client with custom transport and CF disabled."""
custom = httpx.HTTPTransport()
client = sync_client(enable_cf=False, transport=custom)
assert isinstance(client, httpx.Client)
assert client._transport is custom, "When CF is disabled, custom transport should be used directly"
client.close()

6
app/tests/test_package_installer.py
View file

@ -95,7 +95,7 @@ class TestInstalledAndLatest:
mock_version.side_effect = PackageNotFoundError
assert inst._get_installed_version("bar") is None
@patch("app.library.PackageInstaller.httpx.Client")
@patch("app.library.PackageInstaller.sync_client")
def test_get_latest_version_success(self, mock_client, tmp_path: Path) -> None:
inst = PackageInstaller(pkg_path=tmp_path)
@ -108,7 +108,7 @@ class TestInstalledAndLatest:
assert inst._get_latest_version("foo") == "9.9.9"
@patch("app.library.PackageInstaller.httpx.Client")
@patch("app.library.PackageInstaller.sync_client")
def test_get_latest_version_non_200(self, mock_client, tmp_path: Path) -> None:
inst = PackageInstaller(pkg_path=tmp_path)
client = MagicMock()
@ -118,7 +118,7 @@ class TestInstalledAndLatest:
mock_client.return_value.__enter__.return_value = client
assert inst._get_latest_version("foo") is None
@patch("app.library.PackageInstaller.httpx.Client")
@patch("app.library.PackageInstaller.sync_client")
def test_get_latest_version_exception(self, mock_client, tmp_path: Path) -> None:
inst = PackageInstaller(pkg_path=tmp_path)
mock_client.side_effect = RuntimeError("boom")

BIN
sc_short.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 248 KiB

After

Width:  |  Height:  |  Size: 294 KiB

BIN
sc_simple.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 255 KiB

After

Width:  |  Height:  |  Size: 412 KiB

49
ui/app/stores/SocketStore.ts
View file

@ -1,4 +1,6 @@
import { io, type Socket as IOSocket, type SocketOptions, type ManagerOptions } from "socket.io-client"
import { ref, readonly } from 'vue'
import { defineStore } from 'pinia'
import type { ConfigState } from "~/types/config";
import type { StoreItem } from "~/types/store";
@ -15,6 +17,8 @@ export const useSocketStore = defineStore('socket', () => {
const connectionStatus = ref<connectionStatus>('disconnected')
const error = ref<string | null>(null)
const error_count = ref<number>(0)
const wasHidden = ref<boolean>(false)
const reconnectTimeout = ref<NodeJS.Timeout | null>(null)
const emit = (event: string, data?: any): any => socket.value?.emit(event, data)
const on = (event: string | string[], callback: (...args: any[]) => void, withEvent: boolean = false) => {
@ -33,6 +37,46 @@ export const useSocketStore = defineStore('socket', () => {
const getSessionId = (): string | null => socket.value?.id || null
const handleVisibilityChange = () => {
if (document.hidden) {
wasHidden.value = true
return
}
if (true === wasHidden.value && false === isConnected.value) {
if (null !== reconnectTimeout.value) {
clearTimeout(reconnectTimeout.value)
reconnectTimeout.value = null
}
reconnectTimeout.value = setTimeout(() => {
if (false === isConnected.value) {
console.debug('[SocketStore] Page visible after background, reconnecting...')
reconnect()
}
reconnectTimeout.value = null
}, 100)
}
wasHidden.value = false
}
const setupVisibilityListener = () => {
if (typeof document !== 'undefined') {
document.addEventListener('visibilitychange', handleVisibilityChange)
}
}
const cleanupVisibilityListener = () => {
if (typeof document !== 'undefined') {
document.removeEventListener('visibilitychange', handleVisibilityChange)
}
if (null !== reconnectTimeout.value) {
clearTimeout(reconnectTimeout.value)
reconnectTimeout.value = null
}
}
const reconnect = () => {
if (true === isConnected.value) {
return;
@ -49,6 +93,7 @@ export const useSocketStore = defineStore('socket', () => {
socket.value = null;
isConnected.value = false;
connectionStatus.value = 'disconnected';
cleanupVisibilityListener();
}
const connect = () => {
@ -234,6 +279,8 @@ export const useSocketStore = defineStore('socket', () => {
on('presets_update', (data: string) => config.update('presets', JSON.parse(data).data || []));
on('dlfields_update', (data: string) => config.update('dl_fields', JSON.parse(data).data || []));
setupVisibilityListener();
}
if (false === isConnected.value) {
@ -243,7 +290,7 @@ export const useSocketStore = defineStore('socket', () => {
return {
connect, reconnect, disconnect,
on, off, emit,
socket, isConnected,
isConnected,
getSessionId,
connectionStatus: readonly(connectionStatus) as Readonly<Ref<connectionStatus>>,
error: readonly(error) as Readonly<Ref<string | null>>,