Merge pull request #529 from arabcoders/dev

Feat: CF Solver handler

.vscode/settings.json

@@ -167,6 +167,7 @@
     "smhd",
     "socketio",
     "softprops",
+    "solverr",
     "sstr",
     "startswith",
     "SUPPRESSHELP",

FAQ.md

@@ -4,7 +4,7 @@ Certain configuration values can be set via environment variables, using the `-e
 or the `environment:` section in `compose.yaml` file.
 
 | Environment Variable            | Description                                                         | Default               |
-| ------------------------------ | ------------------------------------------------------------------- | --------------------- |
+| ------------------------------- | ------------------------------------------------------------------- | --------------------- |
 | TZ                              | The timezone to use for the application                             | `(not_set)`           |
 | YTP_OUTPUT_TEMPLATE             | The template for the filenames of the downloaded videos             | `%(title)s.%(ext)s`   |
 | YTP_DEFAULT_PRESET              | The default preset to use for the download                          | `default`             |
@@ -38,6 +38,10 @@ or the `environment:` section in `compose.yaml` file.
 | YTP_YTDLP_AUTO_UPDATE           | Whether to enable the auto update for yt-dlp                        | `true`                |
 | YTP_YTDLP_DEBUG                 | Whether to turn debug logging for the internal `yt-dlp` package     | `false`               |
 | YTP_YTDLP_VERSION               | The version of yt-dlp to use. Defaults to latest version            | `(not_set)`           |
+| YTP_FLARESOLVERR_URL            | FlareSolverr endpoint URL.                                          | `(not_set)`           |
+| YTP_FLARESOLVERR_MAX_TIMEOUT    | Max FlareSolverr challenge timeout in seconds                       | `120`                 |
+| YTP_FLARESOLVERR_CLIENT_TIMEOUT | HTTP client timeout (seconds) when calling FlareSolverr             | `120`                 |
+| YTP_FLARESOLVERR_CACHE_TTL      | The cache TTL (in seconds) for FlareSolverr solutions               | `600`                 |
 | YTP_BASE_PATH                   | Set this if you are serving YTPTube from sub-folder                 | `/`                   |
 | YTP_PREVENT_LIVE_PREMIERE       | Prevents the initial youtube premiere stream from being downloaded  | `false`               |
 | YTP_LIVE_PREMIERE_BUFFER        | buffer time in minutes to add to video duration                     | `5`                   |
@@ -569,3 +573,35 @@ YTP_LIVE_PREMIERE_BUFFER=10
 Where `YTP_LIVE_PREMIERE_BUFFER` is the buffer time in minutes to add to the video duration before the download starts. 
 This will help in case the premiere has a longer loading screen than usual.
 
+# How to bypass CF challenges?
+
+You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL` environment variable to point to your FlareSolverr instance. For example:
+
+```yaml
+services:
+  ytptube:
+    user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id, for example: "1000:1000"
+    image: ghcr.io/arabcoders/ytptube:latest
+    container_name: ytptube
+    restart: unless-stopped
+    environment:
+      - YTP_FLARESOLVERR_URL=http://flaresolverr:8191/v1
+    ports:
+      - "8081:8081"
+    volumes:
+      - ./config:/config:rw
+      - ./downloads:/downloads:rw
+    tmpfs:
+      - /tmp
+    depends_on:
+      - flaresolverr
+  flaresolverr:
+    image: flaresolverr/flaresolverr:latest
+    container_name: flaresolverr
+    restart: unless-stopped    
+```
+
+For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project.
+
+> [!NOTE]
+> This will only work for yt-dlp part of the project. Anything else will not be affected by this setting for now.

app/library/CFSolverRH.py

@@ -0,0 +1,309 @@
+# flake8: noqa: S310
+from __future__ import annotations
+
+import http.cookiejar
+import json
+import logging
+import time
+import urllib.request
+from abc import ABC
+from collections.abc import Callable
+from typing import Any, ClassVar
+from urllib.parse import urlparse
+
+from yt_dlp.networking.common import (
+    _REQUEST_HANDLERS,
+    _RH_PREFERENCES,
+    Request,
+    RequestDirector,
+    RequestHandler,
+    Response,
+    register_preference,
+    register_rh,
+)
+from yt_dlp.networking.exceptions import HTTPError
+from yt_dlp.utils.networking import clean_headers
+
+LOG: logging.Logger = logging.getLogger(__name__)
+
+SolverFn = Callable[[Request, Response, RequestHandler], Request | None]
+CacheEntry = dict[str, Any]
+
+
+def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> Request | None:
+    """
+    A Cloudflare solver that uses FlareSolverr/FlareSolverr to solve challenges.
+
+    Args:
+        request (Request): The original request that triggered the challenge.
+        _response (Response): The response that contained the challenge.
+        handler (RequestHandler): The request handler invoking the solver.
+
+    Returns:
+        Request | None: The modified request with solved credentials, or None if solving failed.
+
+    """
+    from app.library.config import Config
+
+    config = Config.get_instance()
+
+    if not config.flaresolverr_url:
+        return None
+
+    parsed_endpoint = urlparse(config.flaresolverr_url)
+    if parsed_endpoint.scheme not in ("http", "https"):
+        return None
+
+    if request.data is not None and request.method not in ("GET", None):
+        return None
+
+    method: str = request.method.lower() if isinstance(request.method, str) else "get"
+    if method not in ("get", "head"):
+        method = "get"
+
+    payload: dict[str, Any] = {
+        "cmd": f"request.{method}",
+        "url": request.url,
+        "maxTimeout": int(getattr(config, "flaresolverr_max_timeout", 60) * 1000),
+    }
+
+    cookiejar = handler._get_cookiejar(request)
+    cookies = (
+        [
+            {
+                "name": cookie.name,
+                "value": cookie.value,
+                "domain": cookie.domain or urlparse(request.url).hostname or "",
+                "path": cookie.path or "/",
+            }
+            for cookie in cookiejar
+        ]
+        if cookiejar
+        else []
+    )
+
+    if cookies:
+        payload["cookies"] = cookies
+
+    req = urllib.request.Request(
+        config.flaresolverr_url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+
+    try:
+        LOG.info(f"Trying to solve Cloudflare challenge for '{request.url}' this may take a while...")
+        with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
+            result = json.loads(resp.read().decode("utf-8"))
+    except Exception as e:
+        LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {e!s}")
+        return None
+
+    if "ok" != result.get("status"):
+        LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {result}")
+        return None
+
+    LOG.info(f"Successfully solved Cloudflare challenge for '{request.url}'.")
+
+    solution = result.get("solution") or {}
+    _cookiejar_from_solution(solution.get("cookies"), request, handler)
+
+    if ua := solution.get("userAgent"):
+        request.headers["User-Agent"] = ua
+
+    CFSolverRH.cache[urlparse(request.url).netloc] = {
+        "cookies": solution.get("cookies") or [],
+        "userAgent": ua,
+        "expires_at": time.time() + config.flaresolverr_cache_ttl,
+    }
+
+    return request
+
+
+def set_cf_handler(solver: SolverFn | None = None) -> type[CFSolverRH]:
+    """
+    Set the Cloudflare handler.
+
+    Args:
+        solver (SolverFn | None): The solver function to use for Cloudflare challenges.
+            If None, the existing solver will be used.
+
+    Returns:
+        type[CloudflareRH]: The Cloudflare request handler class.
+
+    """
+    CFSolverRH.solver = solver or CFSolverRH.solver
+
+    return CFSolverRH
+
+
+def _cookiejar_from_solution(cookies, request: Request, handler: RequestHandler) -> None:
+    cookiejar = handler._get_cookiejar(request)
+    host = urlparse(request.url).hostname or ""
+    for cookie in cookies or []:
+        name = cookie.get("name")
+        value = cookie.get("value")
+        if not name or value is None:
+            continue
+        domain = cookie.get("domain") or host
+        path = cookie.get("path") or "/"
+        cookiejar.set_cookie(
+            http.cookiejar.Cookie(
+                version=0,
+                name=name,
+                value=value,
+                port=None,
+                port_specified=False,
+                domain=domain,
+                domain_specified=True,
+                domain_initial_dot=domain.startswith("."),
+                path=path,
+                path_specified=True,
+                secure=bool(cookie.get("secure")),
+                expires=cookie.get("expires"),
+                discard=False,
+                comment=None,
+                comment_url=None,
+                rest={},
+                rfc2109=False,
+            )
+        )
+
+
+@register_rh
+class CFSolverRH(RequestHandler, ABC):
+    """Request handler that intercepts Cloudflare challenges"""
+
+    _SUPPORTED_URL_SCHEMES = ("http", "https")
+    _SUPPORTED_PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h")
+    solver: ClassVar[SolverFn | None] = None
+    cache: ClassVar[dict[str, CacheEntry]] = {}
+
+    def __init__(self, *, solver: SolverFn | None = None, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self._solver: SolverFn | None = solver or cf_solver
+        self._fallback_director: RequestDirector | None = None
+
+    def close(self) -> None:
+        if self._fallback_director:
+            self._fallback_director.close()
+            self._fallback_director = None
+
+    def _check_extensions(self, extensions) -> None:
+        super()._check_extensions(extensions)
+        for key in ("cookiejar", "timeout", "legacy_ssl", "keep_header_casing", "impersonate", "cf_retry"):
+            extensions.pop(key, None)
+
+    def _build_fallback(self) -> RequestDirector:
+        if self._fallback_director:
+            return self._fallback_director
+
+        director = RequestDirector(logger=self._logger, verbose=self.verbose)
+        for handler_cls in _REQUEST_HANDLERS.values():
+            if handler_cls.RH_KEY == self.RH_KEY:
+                continue
+            director.add_handler(
+                handler_cls(
+                    logger=self._logger,
+                    headers=self.headers,
+                    cookiejar=self.cookiejar,
+                    proxies=self.proxies,
+                    timeout=self.timeout,
+                    source_address=self.source_address,
+                    verbose=self.verbose,
+                    prefer_system_certs=self.prefer_system_certs,
+                    client_cert=self._client_cert,
+                    verify=self.verify,
+                    legacy_ssl_support=self.legacy_ssl_support,
+                )
+            )
+        director.preferences.update(_RH_PREFERENCES)
+        self._fallback_director = director
+        return director
+
+    @staticmethod
+    def _is_cf_response(response: Response) -> bool:
+        """
+        Check if the response is a Cloudflare challenge response.
+
+        Args:
+            response (Response): The HTTP response to check.
+
+        Returns:
+            bool: True if the response is a Cloudflare challenge, False otherwise.
+
+        """
+        status: int | None = getattr(response, "status", None)
+        if status not in (403, 429, 503):
+            return False
+
+        headers = response.headers or {}
+        server_header: str = (headers.get("Server") or "").lower()
+        if "cloudflare" in server_header:
+            return True
+
+        cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor")
+        return any(key in headers for key in cf_header_keys)
+
+    def _solve(self, request: Request, response: Response) -> Request | None:
+        return self._solver(request, response, self) if self._solver else None
+
+    @staticmethod
+    def _mark_retry(request: Request) -> Request:
+        new_request: Request = request.copy()
+        new_request.extensions["cf_retry"] = True
+        return new_request
+
+    def _retry_with_clearance(self, request: Request, response: Response, director: RequestDirector) -> Response:
+        if request.extensions.get("cf_retry"):
+            return response
+
+        solved_request: Request | None = self._solve(self._mark_retry(request), response)
+        if solved_request is None:
+            return response
+
+        solved_request.extensions.pop("cf_retry", None)
+        clean_headers(solved_request.headers)
+        response.close()
+        return director.send(solved_request)
+
+    def _inject_cookie(self, request: Request) -> None:
+        cache_key: str = urlparse(request.url).netloc
+        if not (cached := self.cache.get(cache_key)):
+            return
+
+        if cached.get("expires_at", 0) <= time.time():
+            self.cache.pop(cache_key, None)
+            return
+
+        LOG.info(f"Injecting cached Cloudflare cookies for '{cache_key}'.")
+        _cookiejar_from_solution(cached.get("cookies"), request, self)
+        if ua := cached.get("userAgent"):
+            request.headers["User-Agent"] = ua
+
+    def _send(self, request: Request) -> Response:
+        self._inject_cookie(request)
+        director: RequestDirector = self._build_fallback()
+
+        try:
+            response: Response = director.send(request)
+        except HTTPError as error:
+            if error.response and self._is_cf_response(error.response):
+                return self._retry_with_clearance(request, error.response, director)
+            raise
+
+        if self._is_cf_response(response):
+            return self._retry_with_clearance(request, response, director)
+
+        return response
+
+
+@register_preference(CFSolverRH)
+def cf_solver_preference(_handler: RequestHandler, _request: Request) -> int:
+    from app.library.config import Config
+
+    if not Config.get_instance().flaresolverr_url:
+        return 0
+
+    return 1000

app/library/config.py

@@ -183,6 +183,18 @@ class Config(metaclass=Singleton):
     ytdlp_debug: bool = False
     """Enable yt-dlp debugging."""
 
+    flaresolverr_url: str = ""
+    """FlareSolverr endpoint URL."""
+
+    flaresolverr_max_timeout: int = 120
+    """Max FlareSolverr challenge timeout in seconds."""
+
+    flaresolverr_client_timeout: int = 120
+    """HTTP client timeout (seconds) when calling FlareSolverr."""
+
+    flaresolverr_cache_ttl: int = 600
+    """The cache TTL (in seconds) for FlareSolverr solutions."""
+
     is_native: bool = False
     "Is the application running in natively."
 
@@ -244,6 +256,9 @@ class Config(metaclass=Singleton):
         "download_info_expires",
         "auto_clear_history_days",
         "default_pagination",
+        "flaresolverr_max_timeout",
+        "flaresolverr_client_timeout",
+        "flaresolverr_cache_ttl",
     )
     "The variables that are integers."
 

app/library/ytdlp.py

@@ -4,6 +4,9 @@ from typing import Any
 import yt_dlp
 from yt_dlp.utils import make_archive_id
 
+# imported to register the handler with yt-dlp.
+from app.library.CFSolverRH import set_cf_handler
+
 
 class _ArchiveProxy:
     """

app/routes/api/conditions.py

@@ -1,3 +1,5 @@
+import asyncio
+import functools
 import logging
 import uuid
 from collections import OrderedDict
@@ -153,14 +155,22 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
         preset: str = params.get("preset", config.default_preset)
         key: str = cache.hash(url + str(preset))
         if not cache.has(key):
-            data: dict = extract_info(
+            data: dict | None = await asyncio.wait_for(
+                fut=asyncio.get_running_loop().run_in_executor(
+                    None,
+                    functools.partial(
+                        extract_info,
                         config=YTDLPOpts.get_instance().preset(name=preset).get_all(),
                         url=url,
                         debug=False,
                         no_archive=True,
                         follow_redirect=True,
                         sanitize_info=True,
+                    ),
+                ),
+                timeout=120,
             )
+
             if not data:
                 return web.json_response(
                     data={"error": f"Failed to extract info from '{url!s}'."},

app/routes/api/yt_dlp.py

@@ -1,3 +1,5 @@
+import asyncio
+import functools
 import json
 import logging
 import time
@@ -163,13 +165,20 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
             },
         }
 
-        data = extract_info(
+        data: dict | None = await asyncio.wait_for(
+            fut=asyncio.get_running_loop().run_in_executor(
+                None,
+                functools.partial(
+                    extract_info,
                     config=ytdlp_opts,
                     url=url,
                     debug=False,
                     no_archive=True,
                     follow_redirect=True,
                     sanitize_info=True,
+                ),
+            ),
+            timeout=120,
         )
 
         if not data or not isinstance(data, dict):