From 47be76fd73ae853eea95af9b57933b02e7add695 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Thu, 1 Jan 2026 20:55:57 +0300 Subject: [PATCH] Refactor: add caching for FlareSolverr solutions. --- FAQ.md | 1 + app/library/CFSolverRH.py | 52 ++++++++++++++++++++++++++++----------- app/library/config.py | 4 +++ app/library/ytdlp.py | 2 +- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/FAQ.md b/FAQ.md index b35696de..e9420737 100644 --- a/FAQ.md +++ b/FAQ.md @@ -41,6 +41,7 @@ or the `environment:` section in `compose.yaml` file. | YTP_FLARESOLVERR_URL | FlareSolverr endpoint URL. | `(not_set)` | | YTP_FLARESOLVERR_MAX_TIMEOUT | Max FlareSolverr challenge timeout in seconds | `120` | | YTP_FLARESOLVERR_CLIENT_TIMEOUT | HTTP client timeout (seconds) when calling FlareSolverr | `120` | +| YTP_FLARESOLVERR_CACHE_TTL | The cache TTL (in seconds) for FlareSolverr solutions | `600` | | YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | | YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | | YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` | diff --git a/app/library/CFSolverRH.py b/app/library/CFSolverRH.py index 60bcafc8..fb43a1d8 100644 --- a/app/library/CFSolverRH.py +++ b/app/library/CFSolverRH.py @@ -4,7 +4,9 @@ from __future__ import annotations import http.cookiejar import json import logging +import time import urllib.request +from abc import ABC from collections.abc import Callable from typing import Any, ClassVar from urllib.parse import urlparse @@ -25,6 +27,7 @@ from yt_dlp.utils.networking import clean_headers LOG: logging.Logger = logging.getLogger(__name__) SolverFn = Callable[[Request, Response, RequestHandler], Request | None] +CacheEntry = dict[str, Any] def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> Request | None: @@ -106,10 +109,15 @@ def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> solution = result.get("solution") or {} _cookiejar_from_solution(solution.get("cookies"), request, handler) - ua = solution.get("userAgent") - if ua: + if ua := solution.get("userAgent"): request.headers["User-Agent"] = ua + CFSolverRH.cache[urlparse(request.url).netloc] = { + "cookies": solution.get("cookies") or [], + "userAgent": ua, + "expires_at": time.time() + config.flaresolverr_cache_ttl, + } + return request @@ -163,25 +171,14 @@ def _cookiejar_from_solution(cookies, request: Request, handler: RequestHandler) ) -@register_preference() -def _prefer_cf_handler(handler: RequestHandler, _request: Request) -> int: - """Prefer Cloudflare handler when configured with endpoint.""" - from app.library.config import Config - - if not Config.get_instance().flaresolverr_url: - return 0 - - hand = getattr(handler, "RH_KEY", "") - return 1000 if hand == "CFSolver" else 0 - - @register_rh -class CFSolverRH(RequestHandler): +class CFSolverRH(RequestHandler, ABC): """Request handler that intercepts Cloudflare challenges""" _SUPPORTED_URL_SCHEMES = ("http", "https") _SUPPORTED_PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h") solver: ClassVar[SolverFn | None] = None + cache: ClassVar[dict[str, CacheEntry]] = {} def __init__(self, *, solver: SolverFn | None = None, **kwargs) -> None: super().__init__(**kwargs) @@ -271,7 +268,22 @@ class CFSolverRH(RequestHandler): response.close() return director.send(solved_request) + def _inject_cookie(self, request: Request) -> None: + cache_key: str = urlparse(request.url).netloc + if not (cached := self.cache.get(cache_key)): + return + + if cached.get("expires_at", 0) <= time.time(): + self.cache.pop(cache_key, None) + return + + LOG.info(f"Injecting cached Cloudflare cookies for '{cache_key}'.") + _cookiejar_from_solution(cached.get("cookies"), request, self) + if ua := cached.get("userAgent"): + request.headers["User-Agent"] = ua + def _send(self, request: Request) -> Response: + self._inject_cookie(request) director: RequestDirector = self._build_fallback() try: @@ -285,3 +297,13 @@ class CFSolverRH(RequestHandler): return self._retry_with_clearance(request, response, director) return response + + +@register_preference(CFSolverRH) +def cf_solver_preference(_handler: RequestHandler, _request: Request) -> int: + from app.library.config import Config + + if not Config.get_instance().flaresolverr_url: + return 0 + + return 1000 diff --git a/app/library/config.py b/app/library/config.py index 8c1e2628..b34aaeb7 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -192,6 +192,9 @@ class Config(metaclass=Singleton): flaresolverr_client_timeout: int = 120 """HTTP client timeout (seconds) when calling FlareSolverr.""" + flaresolverr_cache_ttl: int = 600 + """The cache TTL (in seconds) for FlareSolverr solutions.""" + is_native: bool = False "Is the application running in natively." @@ -255,6 +258,7 @@ class Config(metaclass=Singleton): "default_pagination", "flaresolverr_max_timeout", "flaresolverr_client_timeout", + "flaresolverr_cache_ttl", ) "The variables that are integers." diff --git a/app/library/ytdlp.py b/app/library/ytdlp.py index 74d53f3e..6e95166c 100644 --- a/app/library/ytdlp.py +++ b/app/library/ytdlp.py @@ -4,6 +4,7 @@ from typing import Any import yt_dlp from yt_dlp.utils import make_archive_id +# imported to register the handler with yt-dlp. from app.library.CFSolverRH import set_cf_handler @@ -51,7 +52,6 @@ class YTDLP(yt_dlp.YoutubeDL): def __init__(self, params=None, auto_init=True): # Avoid yt-dlp preloading the archive file by stripping the param first - set_cf_handler(None) orig_file = None patched_params = None if params is not None: