diff --git a/.vscode/settings.json b/.vscode/settings.json index 2439e05e..de180aac 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -167,6 +167,7 @@ "smhd", "socketio", "softprops", + "solverr", "sstr", "startswith", "SUPPRESSHELP", diff --git a/FAQ.md b/FAQ.md index dd5813a0..b35696de 100644 --- a/FAQ.md +++ b/FAQ.md @@ -3,54 +3,57 @@ Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in `compose.yaml` file. -| Environment Variable | Description | Default | -| ------------------------------ | ------------------------------------------------------------------- | --------------------- | -| TZ | The timezone to use for the application | `(not_set)` | -| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | -| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | -| YTP_INSTANCE_TITLE | The title of the instance | `(not_set)` | -| YTP_FILE_LOGGING | Whether to log to file | `false` | -| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | -| YTP_MAX_WORKERS | The maximum number of workers to use for downloading | `20` | -| YTP_MAX_WORKERS_PER_EXTRACTOR | The maximum number of concurrent downloads per extractor | `2` | -| YTP_AUTH_USERNAME | Username for basic authentication | `(not_set)` | -| YTP_AUTH_PASSWORD | Password for basic authentication | `(not_set)` | -| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | -| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | -| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | -| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | -| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | -| YTP_HOST | Which IP address to bind to | `0.0.0.0` | -| YTP_PORT | Which port to bind to | `8081` | -| YTP_LOG_LEVEL | Log level | `info` | -| YTP_STREAMER_VCODEC | The video encoding codec, default to GPU and fallback to software | `""` | -| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | -| YTP_VAAPI_DEVICE | The VAAPI device to use for hardware acceleration. | `/dev/dri/renderD128` | -| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | -| YTP_DEBUG | Whether to turn on debug mode | `false` | -| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | -| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | -| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | -| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `(not_set)` | -| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | -| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use | `(not_set)` | -| YTP_BROWSER_CONTROL_ENABLED | Whether to enable the file browser actions | `false` | -| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | -| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | -| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `(not_set)` | -| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | -| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | -| YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` | -| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` | -| YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time | `1` | -| YTP_TEMP_DISABLED | Disable temp files handling. | `false` | -| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` | -| YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` | -| YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` | -| YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` | -| YTP_AUTO_CLEAR_HISTORY_DAYS | Number of days after which completed download history is cleared. | `0` | -| YTP_DEFAULT_PAGINATION | The default number of items per page for history. | `50` | -| YTP_TASK_HANDLER_RANDOM_DELAY | The maximum random delay in seconds before starting a task handler. | `60` | +| Environment Variable | Description | Default | +| ------------------------------- | ------------------------------------------------------------------- | --------------------- | +| TZ | The timezone to use for the application | `(not_set)` | +| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | +| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | +| YTP_INSTANCE_TITLE | The title of the instance | `(not_set)` | +| YTP_FILE_LOGGING | Whether to log to file | `false` | +| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | +| YTP_MAX_WORKERS | The maximum number of workers to use for downloading | `20` | +| YTP_MAX_WORKERS_PER_EXTRACTOR | The maximum number of concurrent downloads per extractor | `2` | +| YTP_AUTH_USERNAME | Username for basic authentication | `(not_set)` | +| YTP_AUTH_PASSWORD | Password for basic authentication | `(not_set)` | +| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | +| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | +| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | +| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | +| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | +| YTP_HOST | Which IP address to bind to | `0.0.0.0` | +| YTP_PORT | Which port to bind to | `8081` | +| YTP_LOG_LEVEL | Log level | `info` | +| YTP_STREAMER_VCODEC | The video encoding codec, default to GPU and fallback to software | `""` | +| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | +| YTP_VAAPI_DEVICE | The VAAPI device to use for hardware acceleration. | `/dev/dri/renderD128` | +| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | +| YTP_DEBUG | Whether to turn on debug mode | `false` | +| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | +| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | +| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | +| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `(not_set)` | +| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | +| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use | `(not_set)` | +| YTP_BROWSER_CONTROL_ENABLED | Whether to enable the file browser actions | `false` | +| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | +| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | +| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `(not_set)` | +| YTP_FLARESOLVERR_URL | FlareSolverr endpoint URL. | `(not_set)` | +| YTP_FLARESOLVERR_MAX_TIMEOUT | Max FlareSolverr challenge timeout in seconds | `120` | +| YTP_FLARESOLVERR_CLIENT_TIMEOUT | HTTP client timeout (seconds) when calling FlareSolverr | `120` | +| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | +| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | +| YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` | +| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` | +| YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time | `1` | +| YTP_TEMP_DISABLED | Disable temp files handling. | `false` | +| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` | +| YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` | +| YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` | +| YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` | +| YTP_AUTO_CLEAR_HISTORY_DAYS | Number of days after which completed download history is cleared. | `0` | +| YTP_DEFAULT_PAGINATION | The default number of items per page for history. | `50` | +| YTP_TASK_HANDLER_RANDOM_DELAY | The maximum random delay in seconds before starting a task handler. | `60` | > [!NOTE] > To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_`. @@ -569,3 +572,35 @@ YTP_LIVE_PREMIERE_BUFFER=10 Where `YTP_LIVE_PREMIERE_BUFFER` is the buffer time in minutes to add to the video duration before the download starts. This will help in case the premiere has a longer loading screen than usual. +# How to bypass CF challenges? + +You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL` environment variable to point to your FlareSolverr instance. For example: + +```yaml +services: + ytptube: + user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id, for example: "1000:1000" + image: ghcr.io/arabcoders/ytptube:latest + container_name: ytptube + restart: unless-stopped + environment: + - YTP_FLARESOLVERR_URL=http://flaresolverr:8191/v1 + ports: + - "8081:8081" + volumes: + - ./config:/config:rw + - ./downloads:/downloads:rw + tmpfs: + - /tmp + depends_on: + - flaresolverr + flaresolverr: + image: flaresolverr/flaresolverr:latest + container_name: flaresolverr + restart: unless-stopped +``` + +For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project. + +> [!NOTE] +> This will only work for yt-dlp part of the project. Anything else will not be affected by this setting for now. diff --git a/app/library/CFSolverRH.py b/app/library/CFSolverRH.py new file mode 100644 index 00000000..60bcafc8 --- /dev/null +++ b/app/library/CFSolverRH.py @@ -0,0 +1,287 @@ +# flake8: noqa: S310 +from __future__ import annotations + +import http.cookiejar +import json +import logging +import urllib.request +from collections.abc import Callable +from typing import Any, ClassVar +from urllib.parse import urlparse + +from yt_dlp.networking.common import ( + _REQUEST_HANDLERS, + _RH_PREFERENCES, + Request, + RequestDirector, + RequestHandler, + Response, + register_preference, + register_rh, +) +from yt_dlp.networking.exceptions import HTTPError +from yt_dlp.utils.networking import clean_headers + +LOG: logging.Logger = logging.getLogger(__name__) + +SolverFn = Callable[[Request, Response, RequestHandler], Request | None] + + +def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> Request | None: + """ + A Cloudflare solver that uses FlareSolverr/FlareSolverr to solve challenges. + + Args: + request (Request): The original request that triggered the challenge. + _response (Response): The response that contained the challenge. + handler (RequestHandler): The request handler invoking the solver. + + Returns: + Request | None: The modified request with solved credentials, or None if solving failed. + + """ + from app.library.config import Config + + config = Config.get_instance() + + if not config.flaresolverr_url: + return None + + parsed_endpoint = urlparse(config.flaresolverr_url) + if parsed_endpoint.scheme not in ("http", "https"): + return None + + if request.data is not None and request.method not in ("GET", None): + return None + + method: str = request.method.lower() if isinstance(request.method, str) else "get" + if method not in ("get", "head"): + method = "get" + + payload: dict[str, Any] = { + "cmd": f"request.{method}", + "url": request.url, + "maxTimeout": int(getattr(config, "flaresolverr_max_timeout", 60) * 1000), + } + + cookiejar = handler._get_cookiejar(request) + cookies = ( + [ + { + "name": cookie.name, + "value": cookie.value, + "domain": cookie.domain or urlparse(request.url).hostname or "", + "path": cookie.path or "/", + } + for cookie in cookiejar + ] + if cookiejar + else [] + ) + + if cookies: + payload["cookies"] = cookies + + req = urllib.request.Request( + config.flaresolverr_url, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + + try: + LOG.info(f"Trying to solve Cloudflare challenge for '{request.url}' this may take a while...") + with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: + result = json.loads(resp.read().decode("utf-8")) + except Exception as e: + LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {e!s}") + return None + + if "ok" != result.get("status"): + LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {result}") + return None + + LOG.info(f"Successfully solved Cloudflare challenge for '{request.url}'.") + + solution = result.get("solution") or {} + _cookiejar_from_solution(solution.get("cookies"), request, handler) + + ua = solution.get("userAgent") + if ua: + request.headers["User-Agent"] = ua + + return request + + +def set_cf_handler(solver: SolverFn | None = None) -> type[CFSolverRH]: + """ + Set the Cloudflare handler. + + Args: + solver (SolverFn | None): The solver function to use for Cloudflare challenges. + If None, the existing solver will be used. + + Returns: + type[CloudflareRH]: The Cloudflare request handler class. + + """ + CFSolverRH.solver = solver or CFSolverRH.solver + + return CFSolverRH + + +def _cookiejar_from_solution(cookies, request: Request, handler: RequestHandler) -> None: + cookiejar = handler._get_cookiejar(request) + host = urlparse(request.url).hostname or "" + for cookie in cookies or []: + name = cookie.get("name") + value = cookie.get("value") + if not name or value is None: + continue + domain = cookie.get("domain") or host + path = cookie.get("path") or "/" + cookiejar.set_cookie( + http.cookiejar.Cookie( + version=0, + name=name, + value=value, + port=None, + port_specified=False, + domain=domain, + domain_specified=True, + domain_initial_dot=domain.startswith("."), + path=path, + path_specified=True, + secure=bool(cookie.get("secure")), + expires=cookie.get("expires"), + discard=False, + comment=None, + comment_url=None, + rest={}, + rfc2109=False, + ) + ) + + +@register_preference() +def _prefer_cf_handler(handler: RequestHandler, _request: Request) -> int: + """Prefer Cloudflare handler when configured with endpoint.""" + from app.library.config import Config + + if not Config.get_instance().flaresolverr_url: + return 0 + + hand = getattr(handler, "RH_KEY", "") + return 1000 if hand == "CFSolver" else 0 + + +@register_rh +class CFSolverRH(RequestHandler): + """Request handler that intercepts Cloudflare challenges""" + + _SUPPORTED_URL_SCHEMES = ("http", "https") + _SUPPORTED_PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h") + solver: ClassVar[SolverFn | None] = None + + def __init__(self, *, solver: SolverFn | None = None, **kwargs) -> None: + super().__init__(**kwargs) + self._solver: SolverFn | None = solver or cf_solver + self._fallback_director: RequestDirector | None = None + + def close(self) -> None: + if self._fallback_director: + self._fallback_director.close() + self._fallback_director = None + + def _check_extensions(self, extensions) -> None: + super()._check_extensions(extensions) + for key in ("cookiejar", "timeout", "legacy_ssl", "keep_header_casing", "impersonate", "cf_retry"): + extensions.pop(key, None) + + def _build_fallback(self) -> RequestDirector: + if self._fallback_director: + return self._fallback_director + + director = RequestDirector(logger=self._logger, verbose=self.verbose) + for handler_cls in _REQUEST_HANDLERS.values(): + if handler_cls.RH_KEY == self.RH_KEY: + continue + director.add_handler( + handler_cls( + logger=self._logger, + headers=self.headers, + cookiejar=self.cookiejar, + proxies=self.proxies, + timeout=self.timeout, + source_address=self.source_address, + verbose=self.verbose, + prefer_system_certs=self.prefer_system_certs, + client_cert=self._client_cert, + verify=self.verify, + legacy_ssl_support=self.legacy_ssl_support, + ) + ) + director.preferences.update(_RH_PREFERENCES) + self._fallback_director = director + return director + + @staticmethod + def _is_cf_response(response: Response) -> bool: + """ + Check if the response is a Cloudflare challenge response. + + Args: + response (Response): The HTTP response to check. + + Returns: + bool: True if the response is a Cloudflare challenge, False otherwise. + + """ + status: int | None = getattr(response, "status", None) + if status not in (403, 429, 503): + return False + + headers = response.headers or {} + server_header: str = (headers.get("Server") or "").lower() + if "cloudflare" in server_header: + return True + + cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor") + return any(key in headers for key in cf_header_keys) + + def _solve(self, request: Request, response: Response) -> Request | None: + return self._solver(request, response, self) if self._solver else None + + @staticmethod + def _mark_retry(request: Request) -> Request: + new_request: Request = request.copy() + new_request.extensions["cf_retry"] = True + return new_request + + def _retry_with_clearance(self, request: Request, response: Response, director: RequestDirector) -> Response: + if request.extensions.get("cf_retry"): + return response + + solved_request: Request | None = self._solve(self._mark_retry(request), response) + if solved_request is None: + return response + + solved_request.extensions.pop("cf_retry", None) + clean_headers(solved_request.headers) + response.close() + return director.send(solved_request) + + def _send(self, request: Request) -> Response: + director: RequestDirector = self._build_fallback() + + try: + response: Response = director.send(request) + except HTTPError as error: + if error.response and self._is_cf_response(error.response): + return self._retry_with_clearance(request, error.response, director) + raise + + if self._is_cf_response(response): + return self._retry_with_clearance(request, response, director) + + return response diff --git a/app/library/config.py b/app/library/config.py index 368fa8dd..8c1e2628 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -183,6 +183,15 @@ class Config(metaclass=Singleton): ytdlp_debug: bool = False """Enable yt-dlp debugging.""" + flaresolverr_url: str = "" + """FlareSolverr endpoint URL.""" + + flaresolverr_max_timeout: int = 120 + """Max FlareSolverr challenge timeout in seconds.""" + + flaresolverr_client_timeout: int = 120 + """HTTP client timeout (seconds) when calling FlareSolverr.""" + is_native: bool = False "Is the application running in natively." @@ -244,6 +253,8 @@ class Config(metaclass=Singleton): "download_info_expires", "auto_clear_history_days", "default_pagination", + "flaresolverr_max_timeout", + "flaresolverr_client_timeout", ) "The variables that are integers." diff --git a/app/library/ytdlp.py b/app/library/ytdlp.py index ccbdc8e5..74d53f3e 100644 --- a/app/library/ytdlp.py +++ b/app/library/ytdlp.py @@ -4,6 +4,8 @@ from typing import Any import yt_dlp from yt_dlp.utils import make_archive_id +from app.library.CFSolverRH import set_cf_handler + class _ArchiveProxy: """ @@ -49,6 +51,7 @@ class YTDLP(yt_dlp.YoutubeDL): def __init__(self, params=None, auto_init=True): # Avoid yt-dlp preloading the archive file by stripping the param first + set_cf_handler(None) orig_file = None patched_params = None if params is not None: