317 lines
10 KiB
Python
317 lines
10 KiB
Python
# flake8: noqa: S310
|
|
from __future__ import annotations
|
|
|
|
import http.cookiejar
|
|
import json
|
|
import logging
|
|
import time
|
|
import urllib.request
|
|
from abc import ABC
|
|
from collections.abc import Callable
|
|
from typing import Any, ClassVar
|
|
from urllib.parse import urlparse
|
|
|
|
from yt_dlp.networking.common import (
|
|
_REQUEST_HANDLERS,
|
|
_RH_PREFERENCES,
|
|
Request,
|
|
RequestDirector,
|
|
RequestHandler,
|
|
Response,
|
|
register_preference,
|
|
register_rh,
|
|
)
|
|
from yt_dlp.networking.exceptions import HTTPError
|
|
from yt_dlp.utils.networking import clean_headers
|
|
|
|
LOG: logging.Logger = logging.getLogger(__name__)
|
|
|
|
SolverFn = Callable[[Request, Response, RequestHandler], Request | None]
|
|
CacheEntry = dict[str, Any]
|
|
|
|
|
|
def cf_solver(request: Request, _response: Response, handler: RequestHandler) -> Request | None:
|
|
"""
|
|
A Cloudflare solver that uses FlareSolverr/FlareSolverr to solve challenges.
|
|
|
|
Args:
|
|
request (Request): The original request that triggered the challenge.
|
|
_response (Response): The response that contained the challenge.
|
|
handler (RequestHandler): The request handler invoking the solver.
|
|
|
|
Returns:
|
|
Request | None: The modified request with solved credentials, or None if solving failed.
|
|
|
|
"""
|
|
from app.library.config import Config
|
|
|
|
config = Config.get_instance()
|
|
|
|
if not config.flaresolverr_url:
|
|
return None
|
|
|
|
parsed_endpoint = urlparse(config.flaresolverr_url)
|
|
if parsed_endpoint.scheme not in ("http", "https"):
|
|
return None
|
|
|
|
if request.data is not None and request.method not in ("GET", None):
|
|
return None
|
|
|
|
method: str = request.method.lower() if isinstance(request.method, str) else "get"
|
|
if method not in ("get", "head"):
|
|
method = "get"
|
|
|
|
payload: dict[str, Any] = {
|
|
"cmd": f"request.{method}",
|
|
"url": request.url,
|
|
"maxTimeout": int(getattr(config, "flaresolverr_max_timeout", 60) * 1000),
|
|
}
|
|
|
|
cookiejar = handler._get_cookiejar(request)
|
|
cookies = (
|
|
[
|
|
{
|
|
"name": cookie.name,
|
|
"value": cookie.value,
|
|
"domain": cookie.domain or urlparse(request.url).hostname or "",
|
|
"path": cookie.path or "/",
|
|
}
|
|
for cookie in cookiejar
|
|
]
|
|
if cookiejar
|
|
else []
|
|
)
|
|
|
|
if cookies:
|
|
payload["cookies"] = cookies
|
|
|
|
req = urllib.request.Request(
|
|
config.flaresolverr_url,
|
|
data=json.dumps(payload).encode("utf-8"),
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST",
|
|
)
|
|
|
|
try:
|
|
LOG.info(f"Trying to solve Cloudflare challenge for '{request.url}' this may take a while...")
|
|
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
|
|
result = json.loads(resp.read().decode("utf-8"))
|
|
except Exception as e:
|
|
LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {e!s}")
|
|
return None
|
|
|
|
if "ok" != result.get("status"):
|
|
LOG.error(f"FlareSolverr failed to solve challenge for '{request.url}': {result}")
|
|
return None
|
|
|
|
LOG.info(f"Successfully solved Cloudflare challenge for '{request.url}'.")
|
|
|
|
solution = result.get("solution") or {}
|
|
_cookiejar_from_solution(solution.get("cookies"), request, handler)
|
|
|
|
if ua := solution.get("userAgent"):
|
|
request.headers["User-Agent"] = ua
|
|
|
|
CFSolverRH.cache[urlparse(request.url).netloc] = {
|
|
"cookies": solution.get("cookies") or [],
|
|
"userAgent": ua,
|
|
"expires_at": time.time() + config.flaresolverr_cache_ttl,
|
|
}
|
|
|
|
return request
|
|
|
|
|
|
def set_cf_handler(solver: SolverFn | None = None) -> type[CFSolverRH]:
|
|
"""
|
|
Set the Cloudflare handler.
|
|
|
|
Args:
|
|
solver (SolverFn | None): The solver function to use for Cloudflare challenges.
|
|
If None, the existing solver will be used.
|
|
|
|
Returns:
|
|
type[CloudflareRH]: The Cloudflare request handler class.
|
|
|
|
"""
|
|
CFSolverRH.solver = solver or CFSolverRH.solver
|
|
|
|
return CFSolverRH
|
|
|
|
|
|
def _cookiejar_from_solution(cookies, request: Request, handler: RequestHandler) -> None:
|
|
cookiejar = handler._get_cookiejar(request)
|
|
host = urlparse(request.url).hostname or ""
|
|
for cookie in cookies or []:
|
|
name = cookie.get("name")
|
|
value = cookie.get("value")
|
|
if not name or value is None:
|
|
continue
|
|
domain = cookie.get("domain") or host
|
|
path = cookie.get("path") or "/"
|
|
cookiejar.set_cookie(
|
|
http.cookiejar.Cookie(
|
|
version=0,
|
|
name=name,
|
|
value=value,
|
|
port=None,
|
|
port_specified=False,
|
|
domain=domain,
|
|
domain_specified=True,
|
|
domain_initial_dot=domain.startswith("."),
|
|
path=path,
|
|
path_specified=True,
|
|
secure=bool(cookie.get("secure")),
|
|
expires=cookie.get("expires"),
|
|
discard=False,
|
|
comment=None,
|
|
comment_url=None,
|
|
rest={},
|
|
rfc2109=False,
|
|
)
|
|
)
|
|
|
|
|
|
@register_rh
|
|
class CFSolverRH(RequestHandler, ABC):
|
|
"""Request handler that intercepts Cloudflare challenges"""
|
|
|
|
_SUPPORTED_URL_SCHEMES = ("http", "https")
|
|
_SUPPORTED_PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h")
|
|
solver: ClassVar[SolverFn | None] = None
|
|
cache: ClassVar[dict[str, CacheEntry]] = {}
|
|
|
|
def __init__(self, *, solver: SolverFn | None = None, **kwargs) -> None:
|
|
super().__init__(**kwargs)
|
|
self._solver: SolverFn | None = solver or cf_solver
|
|
self._fallback_director: RequestDirector | None = None
|
|
|
|
def close(self) -> None:
|
|
if self._fallback_director:
|
|
self._fallback_director.close()
|
|
self._fallback_director = None
|
|
|
|
def _check_extensions(self, extensions) -> None:
|
|
super()._check_extensions(extensions)
|
|
for key in ("cookiejar", "timeout", "legacy_ssl", "keep_header_casing", "cf_retry"):
|
|
extensions.pop(key, None)
|
|
|
|
def _validate(self, request: Request): # type: ignore[override]
|
|
self._check_url_scheme(request)
|
|
self._check_proxies(request.proxies or self.proxies)
|
|
extensions = request.extensions.copy()
|
|
self._check_extensions(extensions)
|
|
|
|
def _build_fallback(self) -> RequestDirector:
|
|
if self._fallback_director:
|
|
return self._fallback_director
|
|
|
|
director = RequestDirector(logger=self._logger, verbose=self.verbose)
|
|
for handler_cls in _REQUEST_HANDLERS.values():
|
|
if handler_cls.RH_KEY == self.RH_KEY:
|
|
continue
|
|
|
|
director.add_handler(
|
|
handler_cls(
|
|
logger=self._logger,
|
|
headers=self.headers,
|
|
cookiejar=self.cookiejar,
|
|
proxies=self.proxies,
|
|
timeout=self.timeout,
|
|
source_address=self.source_address,
|
|
verbose=self.verbose,
|
|
prefer_system_certs=self.prefer_system_certs,
|
|
client_cert=self._client_cert,
|
|
verify=self.verify,
|
|
legacy_ssl_support=self.legacy_ssl_support,
|
|
)
|
|
)
|
|
|
|
director.preferences.update(_RH_PREFERENCES)
|
|
self._fallback_director = director
|
|
return director
|
|
|
|
@staticmethod
|
|
def _is_cf_response(response: Response) -> bool:
|
|
"""
|
|
Check if the response is a Cloudflare challenge response.
|
|
|
|
Args:
|
|
response (Response): The HTTP response to check.
|
|
|
|
Returns:
|
|
bool: True if the response is a Cloudflare challenge, False otherwise.
|
|
|
|
"""
|
|
status: int | None = getattr(response, "status", None)
|
|
if status not in (403, 429, 503):
|
|
return False
|
|
|
|
headers = response.headers or {}
|
|
server_header: str = (headers.get("Server") or "").lower()
|
|
if "cloudflare" in server_header:
|
|
return True
|
|
|
|
cf_header_keys: tuple[str, ...] = ("cf-ray", "cf-chl-bypass", "cf-cache-status", "cf-visitor")
|
|
return any(key in headers for key in cf_header_keys)
|
|
|
|
def _solve(self, request: Request, response: Response) -> Request | None:
|
|
return self._solver(request, response, self) if self._solver else None
|
|
|
|
@staticmethod
|
|
def _mark_retry(request: Request) -> Request:
|
|
new_request: Request = request.copy()
|
|
new_request.extensions["cf_retry"] = True
|
|
return new_request
|
|
|
|
def _retry_with_clearance(self, request: Request, response: Response, director: RequestDirector) -> Response:
|
|
if request.extensions.get("cf_retry"):
|
|
return response
|
|
|
|
solved_request: Request | None = self._solve(self._mark_retry(request), response)
|
|
if solved_request is None:
|
|
return response
|
|
|
|
solved_request.extensions.pop("cf_retry", None)
|
|
clean_headers(solved_request.headers)
|
|
response.close()
|
|
return director.send(solved_request)
|
|
|
|
def _inject_cookie(self, request: Request) -> None:
|
|
cache_key: str = urlparse(request.url).netloc
|
|
if not (cached := self.cache.get(cache_key)):
|
|
return
|
|
|
|
if cached.get("expires_at", 0) <= time.time():
|
|
self.cache.pop(cache_key, None)
|
|
return
|
|
|
|
LOG.info(f"Injecting cached Cloudflare cookies for '{cache_key}'.")
|
|
_cookiejar_from_solution(cached.get("cookies"), request, self)
|
|
if ua := cached.get("userAgent"):
|
|
request.headers["User-Agent"] = ua
|
|
|
|
def _send(self, request: Request) -> Response:
|
|
self._inject_cookie(request)
|
|
director: RequestDirector = self._build_fallback()
|
|
|
|
try:
|
|
response: Response = director.send(request)
|
|
except HTTPError as error:
|
|
if error.response and self._is_cf_response(error.response):
|
|
return self._retry_with_clearance(request, error.response, director)
|
|
raise
|
|
|
|
if self._is_cf_response(response):
|
|
return self._retry_with_clearance(request, response, director)
|
|
|
|
return response
|
|
|
|
|
|
@register_preference(CFSolverRH)
|
|
def cf_solver_preference(_handler: RequestHandler, _request: Request) -> int:
|
|
from app.library.config import Config
|
|
|
|
if not Config.get_instance().flaresolverr_url:
|
|
return 0
|
|
|
|
return 100
|