danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix: update extractor options and improve media polling logic

Browse source
This commit is contained in:
arabcoders 2026-05-08 20:54:41 +03:00
parent 928ddd5b9a
commit 12eb4de83d
3 changed files with 89 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
FAQ.md
View file

@ -665,30 +665,24 @@ YTP_BROWSER_URL=playwright+cdp://chrome:9222/
## yt-dlp usage
To enable the extractor for a preset or task, add the following to the `Command options for yt-dlp` field:
```bash
--use-extractors "browser,default"
```
If you want to set the browser extractor options directly on the yt-dlp side, you can also use `--extractor-args` with `browser:url=...`:
If you want to set the browser extractor options directly on the yt-dlp side, you can also use `--extractor-args` with `generic:url=...`:
### Selenium example
```bash
--use-extractors "browser,default" --extractor-args "browser:url=selenium+http://selenium:4444/wd/hub"
--use-extractors "generic" --extractor-args "generic:url=selenium+http://selenium:4444/wd/hub"
```
### Playwright example
```bash
--use-extractors "browser,default" --extractor-args "browser:url=playwright+ws://playwright:3000/"
--use-extractors "generic" --extractor-args "generic:url=playwright+ws://playwright:3000/"
```
### Playwright CDP example
```bash
--use-extractors "browser,default" --extractor-args "browser:url=playwright+cdp://chrome:9222/"
--use-extractors "generic" --extractor-args "generic:url=playwright+cdp://chrome:9222/"
```
The explicit `--extractor-args` value takes priority over `YTP_BROWSER_URL`.

53
app/tests/test_generic_browser.py
View file

@ -44,6 +44,15 @@ def test_cfg_arg_wins(monkeypatch: pytest.MonkeyPatch) -> None:
assert ie._get_config("url", "YTP_BROWSER_URL") == "selenium+http://arg:4444/wd/hub"
def test_safe_url() -> None:
ie = _make_ie()
assert (
ie._safe_url("playwright+ws://user:pass@10.0.0.6:9222/chrome/playwright?token=abc#frag")
== "playwright+ws://***:***@10.0.0.6:9222/chrome/playwright?***#***"
)
def test_real_extract_env(monkeypatch: pytest.MonkeyPatch) -> None:
ie = _make_ie()
@ -62,7 +71,7 @@ def test_real_extract_invalid_url(monkeypatch: pytest.MonkeyPatch) -> None:
ie._real_extract("https://example.com/watch")
def test_real_extract_logs_connect_failure(monkeypatch: pytest.MonkeyPatch) -> None:
def test_log_connect_fail(monkeypatch: pytest.MonkeyPatch) -> None:
ie = _make_ie()
ie.__wrapped__ = Mock()
ie.__wrapped__._real_extract = Mock(return_value={"id": "fallback"})
@ -87,9 +96,9 @@ def test_real_extract_logs_connect_failure(monkeypatch: pytest.MonkeyPatch) -> N
ie.to_screen.assert_called_once_with("Using remote browser for https://example.com/watch")
def test_real_extract_logs_session_failure(monkeypatch: pytest.MonkeyPatch) -> None:
def test_log_session_fail(monkeypatch: pytest.MonkeyPatch) -> None:
ie = _make_ie()
monkeypatch.setenv("YTP_BROWSER_URL", "playwright+ws://browser")
monkeypatch.setenv("YTP_BROWSER_URL", "playwright+ws://browser/path?token=secret")
session = Mock()
session.goto.side_effect = RuntimeError("page crashed")
@ -110,15 +119,45 @@ def test_real_extract_logs_session_failure(monkeypatch: pytest.MonkeyPatch) -> N
ie.report_warning.assert_called_once_with(
"Browser extractor session failed for url=%r browser_url=%r driver=%s error=%s",
"https://example.com/watch",
"playwright+ws://browser",
"playwright+ws://browser/path?***",
"FakeDriver",
session.goto.side_effect,
)
ie.write_debug.assert_any_call("Selected driver FakeDriver for playwright+ws://browser")
ie.write_debug.assert_any_call("Selected driver FakeDriver for playwright+ws://browser/path?***")
ie.write_debug.assert_any_call("Loading page https://example.com/watch")
def test_real_extract_logs_non_html_page(monkeypatch: pytest.MonkeyPatch) -> None:
def test_log_close_fail(monkeypatch: pytest.MonkeyPatch) -> None:
ie = _make_ie()
monkeypatch.setenv("YTP_BROWSER_URL", "playwright+ws://browser/path?token=secret")
session = Mock()
session.content.return_value = ""
session.get_requests.return_value = []
session.get_media_requests.return_value = []
session.close.side_effect = RuntimeError("close failed")
class FakeDriver:
__name__ = "FakeDriver"
@staticmethod
def connect(ws_url: str, timeout: int | None = None):
return session
monkeypatch.setattr(generic_browser.GenericBrowserIE, "_select_driver", lambda self, ws_url: FakeDriver)
ie._real_extract("https://example.com/watch")
ie.report_warning.assert_called_once_with(
"Browser session close failed for url=%r browser_url=%r driver=%s error=%s",
"https://example.com/watch",
"playwright+ws://browser/path?***",
"FakeDriver",
session.close.side_effect,
)
def test_log_non_html(monkeypatch: pytest.MonkeyPatch) -> None:
ie = _make_ie()
monkeypatch.setenv("YTP_BROWSER_URL", "playwright+ws://browser")
@ -145,7 +184,7 @@ def test_real_extract_logs_non_html_page(monkeypatch: pytest.MonkeyPatch) -> Non
ie.write_debug.assert_any_call("plain text body")
def test_extract_network_formats_logs_no_media() -> None:
def test_no_media() -> None:
ie = _make_ie()
ie.__wrapped__ = Mock()
ie.__wrapped__._real_extract = Mock(return_value={"id": "fallback"})

49
app/yt_dlp_plugins/extractor/generic_browser.py
View file

@ -65,7 +65,10 @@ REQUEST_RESOURCE_TYPES: set[str] = {
API_RESOURCE_TYPES: set[str] = {"fetch", "xhr"}
_MEDIA_CANDIDATE_EXTS = [
POST_MEDIA_POLL_INTERVAL_MS = 250
POST_MEDIA_POLL_ATTEMPTS = 8
MEDIA_CANDIDATE_EXTS: list[str] = [
"m3u8",
"mpd",
"mp4",
@ -78,7 +81,7 @@ _MEDIA_CANDIDATE_EXTS = [
"ogg",
]
_MEDIA_ELEMENT_JS = """() => {
MEDIA_ELEMENT_JS: str = """() => {
const mediaUrls = [];
const seen = new Set();
const addUrl = (url, type) => {
@ -98,7 +101,7 @@ _MEDIA_ELEMENT_JS = """() => {
return mediaUrls;
}"""
_PLAYWRIGHT_PREFIXES: tuple[str, ...] = (
PLAYWRIGHT_PREFIXES: tuple[str, ...] = (
"playwright+ws://",
"playwright+wss://",
"playwright+cdp://",
@ -112,7 +115,7 @@ _PLAYWRIGHT_PREFIXES: tuple[str, ...] = (
def _has_possible_media(requests_list: list[dict]) -> bool:
for req in requests_list:
url_lower = req.get("url", "").lower()
for ext in _MEDIA_CANDIDATE_EXTS:
for ext in MEDIA_CANDIDATE_EXTS:
if f".{ext}" in url_lower or f".{ext}?" in url_lower:
return True
ct = (req.get("response", {}).get("headers", {}).get("content-type", "")).lower()
@ -141,6 +144,12 @@ def _wait_for_network_idle(
raise Exception(msg)
return max(1, min(requested_ms, remaining_ms))
def wait_for_late_media() -> None:
for _ in range(POST_MEDIA_POLL_ATTEMPTS):
if _has_possible_media(requests_list):
return
time.sleep(bounded_timeout_ms(POST_MEDIA_POLL_INTERVAL_MS) / 1000)
wait_fn(bounded_timeout_ms(idle_timeout))
for _ in range(api_poll_attempts):
@ -156,6 +165,7 @@ def _wait_for_network_idle(
return
wait_for_media_fn(bounded_timeout_ms(10000))
wait_for_late_media()
def _build_media_requests(requests_list: list[dict], media_elements: list[dict]) -> list[dict]:
@ -313,7 +323,7 @@ class PlaywrightDriver:
return list(requests_list)
def get_media_requests(self) -> list[dict]:
return _build_media_requests(requests_list, page.evaluate(_MEDIA_ELEMENT_JS))
return _build_media_requests(requests_list, page.evaluate(MEDIA_ELEMENT_JS))
def close(self):
context.close()
@ -416,7 +426,7 @@ class SeleniumDriver:
def get_media_requests(self) -> list[dict]:
try:
return _build_media_requests(requests_list, driver.execute_script(_MEDIA_ELEMENT_JS))
return _build_media_requests(requests_list, driver.execute_script(MEDIA_ELEMENT_JS))
except Exception:
return []
@ -442,12 +452,31 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
return value
def _safe_url(self, browser_url: str) -> str:
try:
parsed = urllib.parse.urlsplit(browser_url)
except Exception:
return browser_url
netloc = parsed.netloc
if parsed.username or parsed.password:
host = parsed.hostname or ""
if parsed.port:
host = f"{host}:{parsed.port}"
netloc = f"***:***@{host}" if host else "***:***"
query = "***" if parsed.query else ""
fragment = "***" if parsed.fragment else ""
return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, query, fragment))
def _real_extract(self, url: str) -> dict[str, Any]:
self._url = url
if not (browser_url := self._get_config("url", "YTP_BROWSER_URL")) or self._failed:
return self.__wrapped__._real_extract(self, url)
safe_url = self._safe_url(browser_url)
video_id: str = self._generic_id(url)
timeout: int | None = self._get_timeout_ms()
self.to_screen(f"Using remote browser for {url}")
@ -459,7 +488,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
)
raise ExtractorError(msg)
self.write_debug(f"Selected driver {driver.__name__} for {browser_url}")
self.write_debug(f"Selected driver {driver.__name__} for {safe_url}")
try:
session = driver.connect(browser_url, timeout)
@ -534,7 +563,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
self.report_warning(
"Browser extractor session failed for url=%r browser_url=%r driver=%s error=%s",
url,
browser_url,
safe_url,
driver.__name__,
e,
)
@ -546,7 +575,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
self.report_warning(
"Browser session close failed for url=%r browser_url=%r driver=%s error=%s",
url,
browser_url,
safe_url,
driver.__name__,
e,
)
@ -555,7 +584,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
if ws_url.startswith(("selenium+http://", "selenium+https://")):
return SeleniumDriver if SeleniumDriver.is_available() else None
if ws_url.startswith(_PLAYWRIGHT_PREFIXES):
if ws_url.startswith(PLAYWRIGHT_PREFIXES):
return PlaywrightDriver if PlaywrightDriver.is_available() else None
msg = (