danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Feat: Added Cloudflare solver handler

Browse source
This commit is contained in:
arabcoders 2026-01-01 20:27:07 +03:00
parent baacf83974
commit 6c1421b3b4
5 changed files with 385 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.vscode/settings.json vendored
View file

@ -167,6 +167,7 @@
"smhd", "smhd",
"socketio", "socketio",
"softprops", "softprops",
"solverr",
"sstr", "sstr",
"startswith", "startswith",
"SUPPRESSHELP", "SUPPRESSHELP",

131
FAQ.md
View file

@ -3,54 +3,57 @@
Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line,
or the `environment:` section in `compose.yaml` file. or the `environment:` section in `compose.yaml` file.
| Environment Variable | Description | Default | | Environment Variable | Description | Default |
| ------------------------------ | ------------------------------------------------------------------- | --------------------- | | ------------------------------- | ------------------------------------------------------------------- | --------------------- |
| TZ | The timezone to use for the application | `(not_set)` | | TZ | The timezone to use for the application | `(not_set)` |
| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | | YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` |
| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | | YTP_DEFAULT_PRESET | The default preset to use for the download | `default` |
| YTP_INSTANCE_TITLE | The title of the instance | `(not_set)` | | YTP_INSTANCE_TITLE | The title of the instance | `(not_set)` |
| YTP_FILE_LOGGING | Whether to log to file | `false` | | YTP_FILE_LOGGING | Whether to log to file | `false` |
| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | | YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` |
| YTP_MAX_WORKERS | The maximum number of workers to use for downloading | `20` | | YTP_MAX_WORKERS | The maximum number of workers to use for downloading | `20` |
| YTP_MAX_WORKERS_PER_EXTRACTOR | The maximum number of concurrent downloads per extractor | `2` | | YTP_MAX_WORKERS_PER_EXTRACTOR | The maximum number of concurrent downloads per extractor | `2` |
| YTP_AUTH_USERNAME | Username for basic authentication | `(not_set)` | | YTP_AUTH_USERNAME | Username for basic authentication | `(not_set)` |
| YTP_AUTH_PASSWORD | Password for basic authentication | `(not_set)` | | YTP_AUTH_PASSWORD | Password for basic authentication | `(not_set)` |
| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | | YTP_CONSOLE_ENABLED | Whether to enable the console | `false` |
| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | | YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` |
| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | | YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` |
| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | | YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` |
| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | | YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` |
| YTP_HOST | Which IP address to bind to | `0.0.0.0` | | YTP_HOST | Which IP address to bind to | `0.0.0.0` |
| YTP_PORT | Which port to bind to | `8081` | | YTP_PORT | Which port to bind to | `8081` |
| YTP_LOG_LEVEL | Log level | `info` | | YTP_LOG_LEVEL | Log level | `info` |
| YTP_STREAMER_VCODEC | The video encoding codec, default to GPU and fallback to software | `""` | | YTP_STREAMER_VCODEC | The video encoding codec, default to GPU and fallback to software | `""` |
| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | | YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` |
| YTP_VAAPI_DEVICE | The VAAPI device to use for hardware acceleration. | `/dev/dri/renderD128` | | YTP_VAAPI_DEVICE | The VAAPI device to use for hardware acceleration. | `/dev/dri/renderD128` |
| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | | YTP_ACCESS_LOG | Whether to log access to the web server | `true` |
| YTP_DEBUG | Whether to turn on debug mode | `false` | | YTP_DEBUG | Whether to turn on debug mode | `false` |
| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | | YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` |
| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | | YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` |
| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | | YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` |
| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `(not_set)` | | YTP_PIP_PACKAGES | A space separated list of pip packages to install | `(not_set)` |
| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | | YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` |
| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use | `(not_set)` | | YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use | `(not_set)` |
| YTP_BROWSER_CONTROL_ENABLED | Whether to enable the file browser actions | `false` | | YTP_BROWSER_CONTROL_ENABLED | Whether to enable the file browser actions | `false` |
| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | | YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` |
| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | | YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` |
| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `(not_set)` | | YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `(not_set)` |
| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | | YTP_FLARESOLVERR_URL | FlareSolverr endpoint URL. | `(not_set)` |
| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | | YTP_FLARESOLVERR_MAX_TIMEOUT | Max FlareSolverr challenge timeout in seconds | `120` |
| YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` | | YTP_FLARESOLVERR_CLIENT_TIMEOUT | HTTP client timeout (seconds) when calling FlareSolverr | `120` |
| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` | | YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` |
| YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time | `1` | | YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` |
| YTP_TEMP_DISABLED | Disable temp files handling. | `false` | | YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` |
| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` | | YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` |
| YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` | | YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time | `1` |
| YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` | | YTP_TEMP_DISABLED | Disable temp files handling. | `false` |
| YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` | | YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` |
| YTP_AUTO_CLEAR_HISTORY_DAYS | Number of days after which completed download history is cleared. | `0` | | YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` |
| YTP_DEFAULT_PAGINATION | The default number of items per page for history. | `50` | | YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` |
| YTP_TASK_HANDLER_RANDOM_DELAY | The maximum random delay in seconds before starting a task handler. | `60` | | YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` |
| YTP_AUTO_CLEAR_HISTORY_DAYS | Number of days after which completed download history is cleared. | `0` |
| YTP_DEFAULT_PAGINATION | The default number of items per page for history. | `50` |
| YTP_TASK_HANDLER_RANDOM_DELAY | The maximum random delay in seconds before starting a task handler. | `60` |
> [!NOTE] > [!NOTE]
> To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`. > To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`.
@ -569,3 +572,35 @@ YTP_LIVE_PREMIERE_BUFFER=10
Where `YTP_LIVE_PREMIERE_BUFFER` is the buffer time in minutes to add to the video duration before the download starts. Where `YTP_LIVE_PREMIERE_BUFFER` is the buffer time in minutes to add to the video duration before the download starts.
This will help in case the premiere has a longer loading screen than usual. This will help in case the premiere has a longer loading screen than usual.
# How to bypass CF challenges?
You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL` environment variable to point to your FlareSolverr instance. For example:
```yaml
services:
ytptube:
user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id, for example: "1000:1000"
image: ghcr.io/arabcoders/ytptube:latest
container_name: ytptube
restart: unless-stopped
environment:
- YTP_FLARESOLVERR_URL=http://flaresolverr:8191/v1
ports:
- "8081:8081"
volumes:
- ./config:/config:rw
- ./downloads:/downloads:rw
tmpfs:
- /tmp
depends_on:
- flaresolverr
flaresolverr:
image: flaresolverr/flaresolverr:latest
container_name: flaresolverr
restart: unless-stopped
```
For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project.
> [!NOTE]
> This will only work for yt-dlp part of the project. Anything else will not be affected by this setting for now.

287
app/library/CFSolverRH.py Normal file
View file

@ -0,0 +1,287 @@
# flake8: noqa: S310
from __future__ import annotations
import http.cookiejar
import json
import logging
import urllib.request
from collections.abc import Callable
from typing import Any, ClassVar
from urllib.parse import urlparse
from yt_dlp.networking.common import (
_REQUEST_HANDLERS,
_RH_PREFERENCES,
Request,
RequestDirector,
RequestHandler,
Response,
register_preference,
register_rh,
)
from yt_dlp.networking.exceptions import HTTPError
from yt_dlp.utils.networking import clean_headers
LOG: logging.Logger = logging.getLogger(__name__)
SolverFn = Callable[[Request, Response, RequestHandler], Request | None]
def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> Request | None:
"""
A Cloudflare solver that uses FlareSolverr/FlareSolverr to solve challenges.
Args:
request (Request): The original request that triggered the challenge.
_response (Response): The response that contained the challenge.
handler (RequestHandler): The request handler invoking the solver.
Returns:
Request | None: The modified request with solved credentials, or None if solving failed.
"""
from app.library.config import Config
config = Config.get_instance()
if not config.flaresolverr_url:
return None
parsed_endpoint = urlparse(config.flaresolverr_url)
if parsed_endpoint.scheme not in ("http", "https"):
return None
if request.data is not None and request.method not in ("GET", None):
return None
method: str = request.method.lower() if isinstance(request.method, str) else "get"
if method not in ("get", "head"):
method = "get"
payload: dict[str, Any] = {
"cmd": f"request.{method}",
"url": request.url,
"maxTimeout": int(getattr(config, "flaresolverr_max_timeout", 60) * 1000),
}
cookiejar = handler._get_cookiejar(request)
cookies = (
[
{
"name": cookie.name,
"value": cookie.value,
"domain": cookie.domain or urlparse(request.url).hostname or "",
"path": cookie.path or "/",
}
for cookie in cookiejar
]
if cookiejar
else []
)
if cookies:
payload["cookies"] = cookies
req = urllib.request.Request(
config.flaresolverr_url,
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
try:
LOG.info(f"Trying to solve Cloudflare challenge for '{request.url}' this may take a while...")
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
result = json.loads(resp.read().decode("utf-8"))
except Exception as e:
LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {e!s}")
return None
if "ok" != result.get("status"):
LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {result}")
return None
LOG.info(f"Successfully solved Cloudflare challenge for '{request.url}'.")
solution = result.get("solution") or {}
_cookiejar_from_solution(solution.get("cookies"), request, handler)
ua = solution.get("userAgent")
if ua:
request.headers["User-Agent"] = ua
return request
def set_cf_handler(solver: SolverFn | None = None) -> type[CFSolverRH]:
"""
Set the Cloudflare handler.
Args:
solver (SolverFn | None): The solver function to use for Cloudflare challenges.
If None, the existing solver will be used.
Returns:
type[CloudflareRH]: The Cloudflare request handler class.
"""
CFSolverRH.solver = solver or CFSolverRH.solver
return CFSolverRH
def _cookiejar_from_solution(cookies, request: Request, handler: RequestHandler) -> None:
cookiejar = handler._get_cookiejar(request)
host = urlparse(request.url).hostname or ""
for cookie in cookies or []:
name = cookie.get("name")
value = cookie.get("value")
if not name or value is None:
continue
domain = cookie.get("domain") or host
path = cookie.get("path") or "/"
cookiejar.set_cookie(
http.cookiejar.Cookie(
version=0,
name=name,
value=value,
port=None,
port_specified=False,
domain=domain,
domain_specified=True,
domain_initial_dot=domain.startswith("."),
path=path,
path_specified=True,
secure=bool(cookie.get("secure")),
expires=cookie.get("expires"),
discard=False,
comment=None,
comment_url=None,
rest={},
rfc2109=False,
)
)
@register_preference()
def _prefer_cf_handler(handler: RequestHandler, _request: Request) -> int:
"""Prefer Cloudflare handler when configured with endpoint."""
from app.library.config import Config
if not Config.get_instance().flaresolverr_url:
return 0
hand = getattr(handler, "RH_KEY", "")
return 1000 if hand == "CFSolver" else 0
@register_rh
class CFSolverRH(RequestHandler):
"""Request handler that intercepts Cloudflare challenges"""
_SUPPORTED_URL_SCHEMES = ("http", "https")
_SUPPORTED_PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h")
solver: ClassVar[SolverFn | None] = None
def __init__(self, *, solver: SolverFn | None = None, **kwargs) -> None:
super().__init__(**kwargs)
self._solver: SolverFn | None = solver or cf_solver
self._fallback_director: RequestDirector | None = None
def close(self) -> None:
if self._fallback_director:
self._fallback_director.close()
self._fallback_director = None
def _check_extensions(self, extensions) -> None:
super()._check_extensions(extensions)
for key in ("cookiejar", "timeout", "legacy_ssl", "keep_header_casing", "impersonate", "cf_retry"):
extensions.pop(key, None)
def _build_fallback(self) -> RequestDirector:
if self._fallback_director:
return self._fallback_director
director = RequestDirector(logger=self._logger, verbose=self.verbose)
for handler_cls in _REQUEST_HANDLERS.values():
if handler_cls.RH_KEY == self.RH_KEY:
continue
director.add_handler(
handler_cls(
logger=self._logger,
headers=self.headers,
cookiejar=self.cookiejar,
proxies=self.proxies,
timeout=self.timeout,
source_address=self.source_address,
verbose=self.verbose,
prefer_system_certs=self.prefer_system_certs,
client_cert=self._client_cert,
verify=self.verify,
legacy_ssl_support=self.legacy_ssl_support,
)
)
director.preferences.update(_RH_PREFERENCES)
self._fallback_director = director
return director
@staticmethod
def _is_cf_response(response: Response) -> bool:
"""
Check if the response is a Cloudflare challenge response.
Args:
response (Response): The HTTP response to check.
Returns:
bool: True if the response is a Cloudflare challenge, False otherwise.
"""
status: int | None = getattr(response, "status", None)
if status not in (403, 429, 503):
return False
headers = response.headers or {}
server_header: str = (headers.get("Server") or "").lower()
if "cloudflare" in server_header:
return True
cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor")
return any(key in headers for key in cf_header_keys)
def _solve(self, request: Request, response: Response) -> Request | None:
return self._solver(request, response, self) if self._solver else None
@staticmethod
def _mark_retry(request: Request) -> Request:
new_request: Request = request.copy()
new_request.extensions["cf_retry"] = True
return new_request
def _retry_with_clearance(self, request: Request, response: Response, director: RequestDirector) -> Response:
if request.extensions.get("cf_retry"):
return response
solved_request: Request | None = self._solve(self._mark_retry(request), response)
if solved_request is None:
return response
solved_request.extensions.pop("cf_retry", None)
clean_headers(solved_request.headers)
response.close()
return director.send(solved_request)
def _send(self, request: Request) -> Response:
director: RequestDirector = self._build_fallback()
try:
response: Response = director.send(request)
except HTTPError as error:
if error.response and self._is_cf_response(error.response):
return self._retry_with_clearance(request, error.response, director)
raise
if self._is_cf_response(response):
return self._retry_with_clearance(request, response, director)
return response

11
app/library/config.py
View file

@ -183,6 +183,15 @@ class Config(metaclass=Singleton):
ytdlp_debug: bool = False ytdlp_debug: bool = False
"""Enable yt-dlp debugging.""" """Enable yt-dlp debugging."""
flaresolverr_url: str = ""
"""FlareSolverr endpoint URL."""
flaresolverr_max_timeout: int = 120
"""Max FlareSolverr challenge timeout in seconds."""
flaresolverr_client_timeout: int = 120
"""HTTP client timeout (seconds) when calling FlareSolverr."""
is_native: bool = False is_native: bool = False
"Is the application running in natively." "Is the application running in natively."
@ -244,6 +253,8 @@ class Config(metaclass=Singleton):
"download_info_expires", "download_info_expires",
"auto_clear_history_days", "auto_clear_history_days",
"default_pagination", "default_pagination",
"flaresolverr_max_timeout",
"flaresolverr_client_timeout",
) )
"The variables that are integers." "The variables that are integers."

3
app/library/ytdlp.py
View file

@ -4,6 +4,8 @@ from typing import Any
import yt_dlp import yt_dlp
from yt_dlp.utils import make_archive_id from yt_dlp.utils import make_archive_id
from app.library.CFSolverRH import set_cf_handler
class _ArchiveProxy: class _ArchiveProxy:
""" """
@ -49,6 +51,7 @@ class YTDLP(yt_dlp.YoutubeDL):
def __init__(self, params=None, auto_init=True): def __init__(self, params=None, auto_init=True):
# Avoid yt-dlp preloading the archive file by stripping the param first # Avoid yt-dlp preloading the archive file by stripping the param first
set_cf_handler(None)
orig_file = None orig_file = None
patched_params = None patched_params = None
if params is not None: if params is not None: