From 12eb4de83d98790373c8d91243461d7036c16482 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Fri, 8 May 2026 20:54:41 +0300 Subject: [PATCH 1/5] fix: update extractor options and improve media polling logic --- FAQ.md | 14 ++--- app/tests/test_generic_browser.py | 53 ++++++++++++++++--- .../extractor/generic_browser.py | 49 +++++++++++++---- 3 files changed, 89 insertions(+), 27 deletions(-) diff --git a/FAQ.md b/FAQ.md index 5cabd966..21fa8c13 100644 --- a/FAQ.md +++ b/FAQ.md @@ -665,30 +665,24 @@ YTP_BROWSER_URL=playwright+cdp://chrome:9222/ ## yt-dlp usage -To enable the extractor for a preset or task, add the following to the `Command options for yt-dlp` field: - -```bash ---use-extractors "browser,default" -``` - -If you want to set the browser extractor options directly on the yt-dlp side, you can also use `--extractor-args` with `browser:url=...`: +If you want to set the browser extractor options directly on the yt-dlp side, you can also use `--extractor-args` with `generic:url=...`: ### Selenium example ```bash ---use-extractors "browser,default" --extractor-args "browser:url=selenium+http://selenium:4444/wd/hub" +--use-extractors "generic" --extractor-args "generic:url=selenium+http://selenium:4444/wd/hub" ``` ### Playwright example ```bash ---use-extractors "browser,default" --extractor-args "browser:url=playwright+ws://playwright:3000/" +--use-extractors "generic" --extractor-args "generic:url=playwright+ws://playwright:3000/" ``` ### Playwright CDP example ```bash ---use-extractors "browser,default" --extractor-args "browser:url=playwright+cdp://chrome:9222/" +--use-extractors "generic" --extractor-args "generic:url=playwright+cdp://chrome:9222/" ``` The explicit `--extractor-args` value takes priority over `YTP_BROWSER_URL`. diff --git a/app/tests/test_generic_browser.py b/app/tests/test_generic_browser.py index e8100dba..94244818 100644 --- a/app/tests/test_generic_browser.py +++ b/app/tests/test_generic_browser.py @@ -44,6 +44,15 @@ def test_cfg_arg_wins(monkeypatch: pytest.MonkeyPatch) -> None: assert ie._get_config("url", "YTP_BROWSER_URL") == "selenium+http://arg:4444/wd/hub" +def test_safe_url() -> None: + ie = _make_ie() + + assert ( + ie._safe_url("playwright+ws://user:pass@10.0.0.6:9222/chrome/playwright?token=abc#frag") + == "playwright+ws://***:***@10.0.0.6:9222/chrome/playwright?***#***" + ) + + def test_real_extract_env(monkeypatch: pytest.MonkeyPatch) -> None: ie = _make_ie() @@ -62,7 +71,7 @@ def test_real_extract_invalid_url(monkeypatch: pytest.MonkeyPatch) -> None: ie._real_extract("https://example.com/watch") -def test_real_extract_logs_connect_failure(monkeypatch: pytest.MonkeyPatch) -> None: +def test_log_connect_fail(monkeypatch: pytest.MonkeyPatch) -> None: ie = _make_ie() ie.__wrapped__ = Mock() ie.__wrapped__._real_extract = Mock(return_value={"id": "fallback"}) @@ -87,9 +96,9 @@ def test_real_extract_logs_connect_failure(monkeypatch: pytest.MonkeyPatch) -> N ie.to_screen.assert_called_once_with("Using remote browser for https://example.com/watch") -def test_real_extract_logs_session_failure(monkeypatch: pytest.MonkeyPatch) -> None: +def test_log_session_fail(monkeypatch: pytest.MonkeyPatch) -> None: ie = _make_ie() - monkeypatch.setenv("YTP_BROWSER_URL", "playwright+ws://browser") + monkeypatch.setenv("YTP_BROWSER_URL", "playwright+ws://browser/path?token=secret") session = Mock() session.goto.side_effect = RuntimeError("page crashed") @@ -110,15 +119,45 @@ def test_real_extract_logs_session_failure(monkeypatch: pytest.MonkeyPatch) -> N ie.report_warning.assert_called_once_with( "Browser extractor session failed for url=%r browser_url=%r driver=%s error=%s", "https://example.com/watch", - "playwright+ws://browser", + "playwright+ws://browser/path?***", "FakeDriver", session.goto.side_effect, ) - ie.write_debug.assert_any_call("Selected driver FakeDriver for playwright+ws://browser") + ie.write_debug.assert_any_call("Selected driver FakeDriver for playwright+ws://browser/path?***") ie.write_debug.assert_any_call("Loading page https://example.com/watch") -def test_real_extract_logs_non_html_page(monkeypatch: pytest.MonkeyPatch) -> None: +def test_log_close_fail(monkeypatch: pytest.MonkeyPatch) -> None: + ie = _make_ie() + monkeypatch.setenv("YTP_BROWSER_URL", "playwright+ws://browser/path?token=secret") + + session = Mock() + session.content.return_value = "" + session.get_requests.return_value = [] + session.get_media_requests.return_value = [] + session.close.side_effect = RuntimeError("close failed") + + class FakeDriver: + __name__ = "FakeDriver" + + @staticmethod + def connect(ws_url: str, timeout: int | None = None): + return session + + monkeypatch.setattr(generic_browser.GenericBrowserIE, "_select_driver", lambda self, ws_url: FakeDriver) + + ie._real_extract("https://example.com/watch") + + ie.report_warning.assert_called_once_with( + "Browser session close failed for url=%r browser_url=%r driver=%s error=%s", + "https://example.com/watch", + "playwright+ws://browser/path?***", + "FakeDriver", + session.close.side_effect, + ) + + +def test_log_non_html(monkeypatch: pytest.MonkeyPatch) -> None: ie = _make_ie() monkeypatch.setenv("YTP_BROWSER_URL", "playwright+ws://browser") @@ -145,7 +184,7 @@ def test_real_extract_logs_non_html_page(monkeypatch: pytest.MonkeyPatch) -> Non ie.write_debug.assert_any_call("plain text body") -def test_extract_network_formats_logs_no_media() -> None: +def test_no_media() -> None: ie = _make_ie() ie.__wrapped__ = Mock() ie.__wrapped__._real_extract = Mock(return_value={"id": "fallback"}) diff --git a/app/yt_dlp_plugins/extractor/generic_browser.py b/app/yt_dlp_plugins/extractor/generic_browser.py index 561adf9e..5d39e2c3 100644 --- a/app/yt_dlp_plugins/extractor/generic_browser.py +++ b/app/yt_dlp_plugins/extractor/generic_browser.py @@ -65,7 +65,10 @@ REQUEST_RESOURCE_TYPES: set[str] = { API_RESOURCE_TYPES: set[str] = {"fetch", "xhr"} -_MEDIA_CANDIDATE_EXTS = [ +POST_MEDIA_POLL_INTERVAL_MS = 250 +POST_MEDIA_POLL_ATTEMPTS = 8 + +MEDIA_CANDIDATE_EXTS: list[str] = [ "m3u8", "mpd", "mp4", @@ -78,7 +81,7 @@ _MEDIA_CANDIDATE_EXTS = [ "ogg", ] -_MEDIA_ELEMENT_JS = """() => { +MEDIA_ELEMENT_JS: str = """() => { const mediaUrls = []; const seen = new Set(); const addUrl = (url, type) => { @@ -98,7 +101,7 @@ _MEDIA_ELEMENT_JS = """() => { return mediaUrls; }""" -_PLAYWRIGHT_PREFIXES: tuple[str, ...] = ( +PLAYWRIGHT_PREFIXES: tuple[str, ...] = ( "playwright+ws://", "playwright+wss://", "playwright+cdp://", @@ -112,7 +115,7 @@ _PLAYWRIGHT_PREFIXES: tuple[str, ...] = ( def _has_possible_media(requests_list: list[dict]) -> bool: for req in requests_list: url_lower = req.get("url", "").lower() - for ext in _MEDIA_CANDIDATE_EXTS: + for ext in MEDIA_CANDIDATE_EXTS: if f".{ext}" in url_lower or f".{ext}?" in url_lower: return True ct = (req.get("response", {}).get("headers", {}).get("content-type", "")).lower() @@ -141,6 +144,12 @@ def _wait_for_network_idle( raise Exception(msg) return max(1, min(requested_ms, remaining_ms)) + def wait_for_late_media() -> None: + for _ in range(POST_MEDIA_POLL_ATTEMPTS): + if _has_possible_media(requests_list): + return + time.sleep(bounded_timeout_ms(POST_MEDIA_POLL_INTERVAL_MS) / 1000) + wait_fn(bounded_timeout_ms(idle_timeout)) for _ in range(api_poll_attempts): @@ -156,6 +165,7 @@ def _wait_for_network_idle( return wait_for_media_fn(bounded_timeout_ms(10000)) + wait_for_late_media() def _build_media_requests(requests_list: list[dict], media_elements: list[dict]) -> list[dict]: @@ -313,7 +323,7 @@ class PlaywrightDriver: return list(requests_list) def get_media_requests(self) -> list[dict]: - return _build_media_requests(requests_list, page.evaluate(_MEDIA_ELEMENT_JS)) + return _build_media_requests(requests_list, page.evaluate(MEDIA_ELEMENT_JS)) def close(self): context.close() @@ -416,7 +426,7 @@ class SeleniumDriver: def get_media_requests(self) -> list[dict]: try: - return _build_media_requests(requests_list, driver.execute_script(_MEDIA_ELEMENT_JS)) + return _build_media_requests(requests_list, driver.execute_script(MEDIA_ELEMENT_JS)) except Exception: return [] @@ -442,12 +452,31 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): return value + def _safe_url(self, browser_url: str) -> str: + try: + parsed = urllib.parse.urlsplit(browser_url) + except Exception: + return browser_url + + netloc = parsed.netloc + if parsed.username or parsed.password: + host = parsed.hostname or "" + if parsed.port: + host = f"{host}:{parsed.port}" + netloc = f"***:***@{host}" if host else "***:***" + + query = "***" if parsed.query else "" + fragment = "***" if parsed.fragment else "" + return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, query, fragment)) + def _real_extract(self, url: str) -> dict[str, Any]: self._url = url if not (browser_url := self._get_config("url", "YTP_BROWSER_URL")) or self._failed: return self.__wrapped__._real_extract(self, url) + safe_url = self._safe_url(browser_url) + video_id: str = self._generic_id(url) timeout: int | None = self._get_timeout_ms() self.to_screen(f"Using remote browser for {url}") @@ -459,7 +488,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): ) raise ExtractorError(msg) - self.write_debug(f"Selected driver {driver.__name__} for {browser_url}") + self.write_debug(f"Selected driver {driver.__name__} for {safe_url}") try: session = driver.connect(browser_url, timeout) @@ -534,7 +563,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): self.report_warning( "Browser extractor session failed for url=%r browser_url=%r driver=%s error=%s", url, - browser_url, + safe_url, driver.__name__, e, ) @@ -546,7 +575,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): self.report_warning( "Browser session close failed for url=%r browser_url=%r driver=%s error=%s", url, - browser_url, + safe_url, driver.__name__, e, ) @@ -555,7 +584,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): if ws_url.startswith(("selenium+http://", "selenium+https://")): return SeleniumDriver if SeleniumDriver.is_available() else None - if ws_url.startswith(_PLAYWRIGHT_PREFIXES): + if ws_url.startswith(PLAYWRIGHT_PREFIXES): return PlaywrightDriver if PlaywrightDriver.is_available() else None msg = ( From a0b50ac3517d20c61dae0802950c6dc37a83787b Mon Sep 17 00:00:00 2001 From: arabcoders Date: Fri, 8 May 2026 21:39:12 +0300 Subject: [PATCH 2/5] fix: use yt-dlp meta extractors methods instead of local rolled ones --- app/tests/test_generic_browser.py | 32 +++++++++++++++++ .../extractor/generic_browser.py | 34 +++---------------- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/app/tests/test_generic_browser.py b/app/tests/test_generic_browser.py index 94244818..0ef657e8 100644 --- a/app/tests/test_generic_browser.py +++ b/app/tests/test_generic_browser.py @@ -184,6 +184,38 @@ def test_log_non_html(monkeypatch: pytest.MonkeyPatch) -> None: ie.write_debug.assert_any_call("plain text body") +def test_html_meta(monkeypatch: pytest.MonkeyPatch) -> None: + ie = _make_ie() + monkeypatch.setenv("YTP_BROWSER_URL", "playwright+ws://browser") + ie._looks_like_html = lambda webpage: True + ie._generic_title = generic_browser.GenericIE._generic_title.__get__(ie, generic_browser.GenericBrowserIE) + + session = Mock() + session.content.return_value = ( + '' + '' + '' + "Page Title" + ) + session.get_requests.return_value = [] + session.get_media_requests.return_value = [] + + class FakeDriver: + __name__ = "FakeDriver" + + @staticmethod + def connect(ws_url: str, timeout: int | None = None): + return session + + monkeypatch.setattr(generic_browser.GenericBrowserIE, "_select_driver", lambda self, ws_url: FakeDriver) + + result = ie._real_extract("https://example.com/watch") + + assert result["title"] == "OG Title" + assert result["description"] == "Meta Desc" + assert result["thumbnail"] == "https://img.example/thumb.jpg" + + def test_no_media() -> None: ie = _make_ie() ie.__wrapped__ = Mock() diff --git a/app/yt_dlp_plugins/extractor/generic_browser.py b/app/yt_dlp_plugins/extractor/generic_browser.py index 5d39e2c3..b67e00e7 100644 --- a/app/yt_dlp_plugins/extractor/generic_browser.py +++ b/app/yt_dlp_plugins/extractor/generic_browser.py @@ -542,8 +542,10 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): } if webpage and self._looks_like_html(webpage): - info_dict["title"] = self._get_title(webpage) or self._generic_title(url, webpage) - info_dict["description"] = self._get_desc(webpage) + info_dict["title"] = self._generic_title(url, webpage) + info_dict["description"] = self._og_search_description(webpage, default=None) or self._html_search_meta( + "description", webpage, default=None + ) thumbnail = self._og_search_thumbnail(webpage, default=None) if thumbnail: info_dict["thumbnail"] = thumbnail @@ -776,34 +778,6 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): ) ) - def _get_title(self, webpage: str) -> str | None: - return self._extract_from_html( - webpage, - [ - r"([^<]+)", - r']+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']', - r']+name=["\']title["\'][^>]+content=["\']([^"\']+)["\']', - r"]*>([^<]+)", - ], - ) - - def _get_desc(self, webpage: str) -> str | None: - return self._extract_from_html( - webpage, - [ - r']+property=["\']og:description["\'][^>]+content=["\']([^"\']+)["\']', - r']+name=["\']description["\'][^>]+content=["\']([^"\']+)["\']', - r']*class=["\']description["\'][^>]*>([^<]+)

', - ], - ) - - def _extract_from_html(self, webpage: str, patterns: list[str]) -> str | None: - for pattern in patterns: - match = re.search(pattern, webpage, re.IGNORECASE) - if match and match.group(1): - return match.group(1).strip() - return None - def _get_timeout_ms(self) -> int | None: socket_timeout = self._downloader.params.get("socket_timeout") if isinstance(socket_timeout, (int, float)) and socket_timeout > 0: From 56dbf6fc2d9897380a8b77b5ad43001e8a7e13aa Mon Sep 17 00:00:00 2001 From: arabcoders Date: Fri, 8 May 2026 21:56:41 +0300 Subject: [PATCH 3/5] refactor: update test functions to use tmp_path for archive file handling --- .../definitions/tests/test_rss_handler.py | 30 ++++---- .../definitions/tests/test_tver_handler.py | 12 ++-- .../definitions/tests/test_twitch_handler.py | 8 ++- .../definitions/tests/test_youtube_handler.py | 8 ++- app/features/ytdlp/tests/test_ytdlp_module.py | 46 +++++++------ app/features/ytdlp/tests/test_ytdlp_opts.py | 12 ++-- app/tests/test_condition_ignore.py | 9 +-- app/tests/test_itemdto.py | 27 +++++--- app/tests/test_nfo_maker.py | 68 +++++-------------- 9 files changed, 112 insertions(+), 108 deletions(-) diff --git a/app/features/tasks/definitions/tests/test_rss_handler.py b/app/features/tasks/definitions/tests/test_rss_handler.py index 04333094..e9a27390 100644 --- a/app/features/tasks/definitions/tests/test_rss_handler.py +++ b/app/features/tasks/definitions/tests/test_rss_handler.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from app.features.tasks.definitions.handlers.rss import RssGenericHandler @@ -21,6 +23,10 @@ class DummyOpts: return self._data +def _opts(tmp_path: Path) -> DummyOpts: + return DummyOpts({"download_archive": str(tmp_path / "archive.txt")}) + + class TestRssHandlerParsing: """Test URL parsing for RSS/Atom feeds using the tests() method.""" @@ -47,7 +53,7 @@ class TestRssHandlerExtraction: """Test RSS feed extraction and parsing.""" @pytest.mark.asyncio - async def test_rss_atom_feed_extraction(self, monkeypatch): + async def test_rss_atom_feed_extraction(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Test extraction from Atom feed.""" atom_feed = """ @@ -69,7 +75,7 @@ class TestRssHandlerExtraction: return DummyResponse(atom_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005 task = HandleTask( id=1, @@ -89,7 +95,7 @@ class TestRssHandlerExtraction: assert result.metadata["entry_count"] == 2 @pytest.mark.asyncio - async def test_rss_feed_extraction(self, monkeypatch): + async def test_rss_feed_extraction(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Test extraction from RSS feed.""" rss_feed = """ @@ -113,7 +119,7 @@ class TestRssHandlerExtraction: return DummyResponse(rss_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005 task = HandleTask( id=1, @@ -133,7 +139,7 @@ class TestRssHandlerExtraction: assert result.metadata["entry_count"] == 2 @pytest.mark.asyncio - async def test_can_handle(self, monkeypatch): + async def test_can_handle(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Test can_handle method.""" atom_feed = """ @@ -147,7 +153,7 @@ class TestRssHandlerExtraction: return DummyResponse(atom_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005 task = HandleTask( id=1, @@ -172,7 +178,7 @@ class TestRssHandlerEdgeCases: """Test edge cases in RSS handling.""" @pytest.mark.asyncio - async def test_empty_feed(self, monkeypatch): + async def test_empty_feed(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Test handling of empty feed.""" empty_feed = """ @@ -186,7 +192,7 @@ class TestRssHandlerEdgeCases: return DummyResponse(empty_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005 task = HandleTask( id=1, @@ -202,7 +208,7 @@ class TestRssHandlerEdgeCases: assert result.metadata["entry_count"] == 0 @pytest.mark.asyncio - async def test_invalid_feed_url(self, monkeypatch): + async def test_invalid_feed_url(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Test handling of invalid feed URL.""" from app.features.tasks.definitions.results import TaskFailure @@ -211,7 +217,7 @@ class TestRssHandlerEdgeCases: raise Exception(msg) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005 task = HandleTask( id=1, @@ -225,7 +231,7 @@ class TestRssHandlerEdgeCases: assert isinstance(result, TaskFailure) @pytest.mark.asyncio - async def test_missing_urls_in_feed(self, monkeypatch): + async def test_missing_urls_in_feed(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Test handling of entries missing URLs.""" feed = """ @@ -245,7 +251,7 @@ class TestRssHandlerEdgeCases: return DummyResponse(feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005 task = HandleTask( id=1, diff --git a/app/features/tasks/definitions/tests/test_tver_handler.py b/app/features/tasks/definitions/tests/test_tver_handler.py index d5711728..8fb5a19d 100644 --- a/app/features/tasks/definitions/tests/test_tver_handler.py +++ b/app/features/tasks/definitions/tests/test_tver_handler.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from app.features.tasks.definitions.handlers.tver import TverHandler @@ -25,7 +27,7 @@ class DummyOpts: @pytest.mark.asyncio -async def test_tver_handler_extract(monkeypatch): +async def test_tver_handler_extract(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Test tver handler extraction of episodes from series.""" session_response = { "result": { @@ -118,8 +120,10 @@ async def test_tver_handler_extract(monkeypatch): msg = f"Unexpected URL: {url}" raise RuntimeError(msg) + archive_path = tmp_path / "archive.txt" + monkeypatch.setattr(TverHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"})) + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": str(archive_path)})) task = HandleTask(id=1, name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default") @@ -142,7 +146,7 @@ async def test_tver_handler_extract(monkeypatch): @pytest.mark.parametrize(("url", "should_match"), TverHandler.tests()) -def test_parse(url: str, should_match: bool): +def test_parse(url: str, should_match: bool) -> None: """Test tver URL parsing.""" result = TverHandler.parse(url) if should_match: @@ -153,7 +157,7 @@ def test_parse(url: str, should_match: bool): @pytest.mark.asyncio -async def test_can_handle(): +async def test_can_handle() -> None: """Test tver handler can_handle method.""" task_valid = HandleTask(id=1, name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default") task_invalid = HandleTask(id=2, name="Test", url="https://youtube.com/watch?v=123", preset="default") diff --git a/app/features/tasks/definitions/tests/test_twitch_handler.py b/app/features/tasks/definitions/tests/test_twitch_handler.py index 642c751c..8b3e5e1c 100644 --- a/app/features/tasks/definitions/tests/test_twitch_handler.py +++ b/app/features/tasks/definitions/tests/test_twitch_handler.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from app.features.tasks.definitions.handlers.twitch import TwitchHandler @@ -22,7 +24,7 @@ class DummyOpts: @pytest.mark.asyncio -async def test_twitch_handler_inspect(monkeypatch): +async def test_twitch_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: feed = """ @@ -39,8 +41,10 @@ async def test_twitch_handler_inspect(monkeypatch): async def fake_request(**kwargs): # noqa: ARG001 return DummyResponse(feed) + archive_path = tmp_path / "archive.txt" + monkeypatch.setattr(TwitchHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": str(archive_path)})) # noqa: ARG005 task = HandleTask( id=1, diff --git a/app/features/tasks/definitions/tests/test_youtube_handler.py b/app/features/tasks/definitions/tests/test_youtube_handler.py index 3482eefa..fb9d1ee7 100644 --- a/app/features/tasks/definitions/tests/test_youtube_handler.py +++ b/app/features/tasks/definitions/tests/test_youtube_handler.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from app.features.tasks.definitions.handlers.youtube import YoutubeHandler @@ -22,7 +24,7 @@ class DummyOpts: @pytest.mark.asyncio -async def test_youtube_handler_inspect(monkeypatch): +async def test_youtube_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: feed = """ @@ -41,8 +43,10 @@ async def test_youtube_handler_inspect(monkeypatch): async def fake_request(**kwargs): # noqa: ARG001 return DummyResponse(feed) + archive_path = tmp_path / "archive.txt" + monkeypatch.setattr(YoutubeHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": str(archive_path)})) # noqa: ARG005 task = HandleTask( id=1, diff --git a/app/features/ytdlp/tests/test_ytdlp_module.py b/app/features/ytdlp/tests/test_ytdlp_module.py index de92f599..535de774 100644 --- a/app/features/ytdlp/tests/test_ytdlp_module.py +++ b/app/features/ytdlp/tests/test_ytdlp_module.py @@ -1,4 +1,5 @@ import importlib +from pathlib import Path from unittest.mock import MagicMock, Mock, patch import pytest @@ -10,8 +11,12 @@ from app.features.ytdlp.utils import _DATA from app.features.ytdlp.ytdlp import YTDLP, _ArchiveProxy, ytdlp_options +def _archive_path(tmp_path: Path) -> str: + return str(tmp_path / "archive.txt") + + class TestArchiveProxy: - def test_bool_and_falsey_cases(self) -> None: + def test_bool_and_falsey_cases(self, tmp_path: Path) -> None: # No file path means proxy is falsey and operations return False p = _ArchiveProxy(file=None) assert bool(p) is False @@ -19,22 +24,24 @@ class TestArchiveProxy: assert p.add("id") is False # Empty item also returns False - p2 = _ArchiveProxy(file="/tmp/archive.txt") + p2 = _ArchiveProxy(file=_archive_path(tmp_path)) assert bool(p2) is True assert ("" in p2) is False assert p2.add("") is False @patch("app.features.ytdlp.archiver.Archiver.get_instance") - def test_delegates_to_archiver(self, mock_get_instance) -> None: + def test_delegates_to_archiver(self, mock_get_instance, tmp_path: Path) -> None: arch = MagicMock() mock_get_instance.return_value = arch - p = _ArchiveProxy(file="/tmp/archive.txt") + file = _archive_path(tmp_path) + + p = _ArchiveProxy(file=file) # contains -> read(file, [item]) and check membership arch.read.return_value = ["abc"] assert ("abc" in p) is True - arch.read.assert_called_with("/tmp/archive.txt", ["abc"]) + arch.read.assert_called_with(file, ["abc"]) arch.read.return_value = [] assert ("xyz" in p) is False @@ -42,7 +49,7 @@ class TestArchiveProxy: # add -> add(file, [item]) returns boolean arch.add.return_value = True assert p.add("abc") is True - arch.add.assert_called_with("/tmp/archive.txt", ["abc"]) + arch.add.assert_called_with(file, ["abc"]) arch.add.return_value = False assert p.add("xyz") is False @@ -96,11 +103,12 @@ class TestYTDLP: return ytdlp @patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__") - def test_init_archive_param(self, mock_super_init) -> None: + def test_init_archive_param(self, mock_super_init, tmp_path: Path) -> None: """Test that __init__ removes download_archive before calling super, then restores it.""" mock_super_init.return_value = None - params = {"download_archive": "/tmp/archive.txt", "quiet": True} + file = _archive_path(tmp_path) + params = {"download_archive": file, "quiet": True} ytdlp = YTDLP(params=params, auto_init=True) # Set params as it would be after super().__init__ @@ -114,13 +122,13 @@ class TestYTDLP: assert call_kwargs["auto_init"] is True # Our __init__ code manually sets these after super() - ytdlp.params["download_archive"] = "/tmp/archive.txt" + ytdlp.params["download_archive"] = file - assert ytdlp.params["download_archive"] == "/tmp/archive.txt", "Verify download_archive was restored to params" + assert ytdlp.params["download_archive"] == file, "Verify download_archive was restored to params" # Verify archive proxy was set up assert isinstance(ytdlp.archive, _ArchiveProxy) - assert ytdlp.archive._file == "/tmp/archive.txt" + assert ytdlp.archive._file == file @patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__") def test_init_no_archive(self, mock_super_init) -> None: @@ -251,9 +259,9 @@ class TestYTDLP: # Should not interact with archive ytdlp.archive.add.assert_not_called() - def test_record_archive_adds_id(self) -> None: + def test_record_archive_adds_id(self, tmp_path: Path) -> None: """Test record_download_archive adds the archive ID.""" - ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"}) + ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)}) ytdlp.write_debug = Mock() ytdlp.archive = Mock() ytdlp._make_archive_id = Mock(return_value="youtube test123") @@ -269,9 +277,9 @@ class TestYTDLP: ytdlp.archive.add.assert_called_once_with("youtube test123") ytdlp.write_debug.assert_called_with("Adding to archive: youtube test123") - def test_record_archive_old_ids(self) -> None: + def test_record_archive_old_ids(self, tmp_path: Path) -> None: """Test record_download_archive adds _old_archive_ids when present.""" - ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"}) + ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)}) ytdlp.write_debug = Mock() ytdlp.archive = Mock() ytdlp._make_archive_id = Mock(return_value="youtube new123") @@ -296,9 +304,9 @@ class TestYTDLP: # Should not add duplicate (youtube new123 appears only once) assert calls.count("youtube new123") == 1 - def test_record_archive_empty_old_ids(self) -> None: + def test_record_archive_empty_old_ids(self, tmp_path: Path) -> None: """Test record_download_archive handles empty or invalid _old_archive_ids.""" - ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"}) + ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)}) ytdlp.write_debug = Mock() ytdlp.archive = Mock() ytdlp._make_archive_id = Mock(return_value="youtube test123") @@ -322,9 +330,9 @@ class TestYTDLP: ytdlp.record_download_archive(info_dict) assert ytdlp.archive.add.call_count == 1 # Only main ID - def test_record_archive_empty_id(self) -> None: + def test_record_archive_empty_id(self, tmp_path: Path) -> None: """Test record_download_archive returns early when _make_archive_id returns empty.""" - ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"}) + ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)}) ytdlp.archive = Mock() ytdlp._make_archive_id = Mock(return_value=None) diff --git a/app/features/ytdlp/tests/test_ytdlp_opts.py b/app/features/ytdlp/tests/test_ytdlp_opts.py index 3bc15451..3dcc40f8 100644 --- a/app/features/ytdlp/tests/test_ytdlp_opts.py +++ b/app/features/ytdlp/tests/test_ytdlp_opts.py @@ -1,5 +1,6 @@ """Tests for YTDLPOpts class.""" +from pathlib import Path from unittest.mock import Mock, patch import pytest @@ -912,7 +913,7 @@ class TestYTDLPCli: @patch("app.features.presets.service.Presets") @patch("app.features.ytdlp.ytdlp_opts.create_cookies_file") @patch("app.features.ytdlp.ytdlp_opts.Config") - def test_build_with_cookies_from_user(self, mock_config, mock_create_cookies, mock_presets): + def test_build_with_cookies_from_user(self, mock_config, mock_create_cookies, mock_presets, tmp_path: Path): """Test build with cookies from user.""" from app.library.ItemDTO import Item from app.features.ytdlp.ytdlp_opts import YTDLPCli @@ -920,11 +921,13 @@ class TestYTDLPCli: mock_config_instance = Mock() mock_config_instance.download_path = "/downloads" mock_config_instance.output_template = "%(title)s.%(ext)s" + mock_config_instance.temp_path = str(tmp_path) mock_config_instance.get_replacers.return_value = {} mock_config.get_instance.return_value = mock_config_instance mock_cookie_path = Mock() - mock_cookie_path.__str__ = Mock(return_value="/tmp/cookies.txt") + cookie_path = tmp_path / "cookies.txt" + mock_cookie_path.__str__ = Mock(return_value=str(cookie_path)) mock_create_cookies.return_value = mock_cookie_path mock_presets.get_instance.return_value.get.return_value = None @@ -936,8 +939,8 @@ class TestYTDLPCli: mock_create_cookies.assert_called_once_with("session_id=abc123") assert "--cookies" in command - assert "/tmp/cookies.txt" in command - assert info["merged"]["cookie_file"] == "/tmp/cookies.txt" + assert str(cookie_path) in command + assert info["merged"]["cookie_file"] == str(cookie_path) @patch("app.features.presets.service.Presets") @patch("app.features.ytdlp.ytdlp_opts.Config") @@ -950,6 +953,7 @@ class TestYTDLPCli: mock_config_instance = Mock() mock_config_instance.download_path = "/downloads" mock_config_instance.output_template = "%(title)s.%(ext)s" + mock_config_instance.temp_path = "/tmp" mock_config_instance.get_replacers.return_value = {} mock_config.get_instance.return_value = mock_config_instance diff --git a/app/tests/test_condition_ignore.py b/app/tests/test_condition_ignore.py index a73af683..2962df80 100644 --- a/app/tests/test_condition_ignore.py +++ b/app/tests/test_condition_ignore.py @@ -1,3 +1,4 @@ +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch @@ -67,9 +68,9 @@ class TestConditionIgnoreMatching: class TestConditionIgnorePropagation: @pytest.mark.asyncio - async def test_add_passes_ignore(self) -> None: + async def test_add_passes_ignore(self, tmp_path: Path) -> None: queue = Mock() - queue.config = Mock(temp_path="/tmp", ignore_archived_items=False, ytdlp_debug=False) + queue.config = Mock(temp_path=str(tmp_path), ignore_archived_items=False, ytdlp_debug=False) queue._notify = Mock() item = Item( @@ -102,9 +103,9 @@ class TestConditionIgnorePropagation: assert result == {"status": "ok"} @pytest.mark.asyncio - async def test_add_coerces_ignore(self) -> None: + async def test_add_coerces_ignore(self, tmp_path: Path) -> None: queue = Mock() - queue.config = Mock(temp_path="/tmp", ignore_archived_items=False, ytdlp_debug=False) + queue.config = Mock(temp_path=str(tmp_path), ignore_archived_items=False, ytdlp_debug=False) queue._notify = Mock() item = Item( diff --git a/app/tests/test_itemdto.py b/app/tests/test_itemdto.py index aef8ff14..a7c63498 100644 --- a/app/tests/test_itemdto.py +++ b/app/tests/test_itemdto.py @@ -7,6 +7,10 @@ import pytest from app.library.ItemDTO import Item, ItemDTO +def _archive_path(tmp_path: Path) -> str: + return str(tmp_path / "archive.txt") + + class TestItemFormatAndBasics: @patch("app.features.presets.service.Presets.get_instance") def test_format_validates_and_normalizes(self, mock_presets_get): @@ -71,8 +75,9 @@ class TestItemFormatAndBasics: assert json.loads(item.json())["url"] == "https://example.com" @patch("app.library.ItemDTO.get_archive_id") - def test_item_archive_id_and_is_archived(self, mock_get_id): + def test_item_archive_id_and_is_archived(self, mock_get_id, tmp_path: Path): mock_get_id.return_value = {"archive_id": "x", "id": "x", "ie_key": "k"} + file = _archive_path(tmp_path) # get_archive_id item = Item(url="https://example.com") @@ -85,7 +90,7 @@ class TestItemFormatAndBasics: ): mock_opts.get_instance.return_value.preset.return_value = mock_opts.get_instance.return_value mock_opts.get_instance.return_value.add_cli.return_value = mock_opts.get_instance.return_value - mock_opts.get_instance.return_value.get_all.return_value = {"download_archive": "/tmp/archive.txt"} + mock_opts.get_instance.return_value.get_all.return_value = {"download_archive": file} mock_read.return_value = ["x"] assert item.is_archived() is True @@ -95,30 +100,32 @@ class TestItemDTO: @patch("app.library.ItemDTO.get_archive_id") @patch("app.library.ItemDTO.YTDLPOpts") @patch("app.library.ItemDTO.archive_read") - def test_post_init_sets_archive_flags(self, mock_read, mock_opts, mock_get_id): + def test_post_init_sets_archive_flags(self, mock_read, mock_opts, mock_get_id, tmp_path: Path): # Setup archive id and archive file mock_get_id.return_value = {"archive_id": "arch", "id": "arch", "ie_key": "YT"} mock_opts.get_instance.return_value.preset.return_value = mock_opts.get_instance.return_value mock_opts.get_instance.return_value.add_cli.return_value = mock_opts.get_instance.return_value - mock_opts.get_instance.return_value.get_all.return_value = {"download_archive": "/tmp/a.txt"} + file = _archive_path(tmp_path) + mock_opts.get_instance.return_value.get_all.return_value = {"download_archive": file} mock_read.return_value = ["arch"] dto = ItemDTO(id="vid", title="t", url="u", folder="f") assert dto.archive_id == "arch" - assert dto._archive_file == "/tmp/a.txt" + assert dto._archive_file == file assert dto.is_archivable is True assert dto.is_archived is True @patch("app.library.ItemDTO.get_archive_id") @patch("app.library.ItemDTO.YTDLPOpts") @patch("app.library.ItemDTO.archive_read") - def test_post_init_keeps_skipped(self, mock_read, mock_opts, mock_get_id): + def test_post_init_keeps_skipped(self, mock_read, mock_opts, mock_get_id, tmp_path: Path): mock_get_id.return_value = {"archive_id": "arch", "id": "arch", "ie_key": "YT"} mock_opts.get_instance.return_value.preset.return_value = mock_opts.get_instance.return_value mock_opts.get_instance.return_value.add_cli.return_value = mock_opts.get_instance.return_value + file = _archive_path(tmp_path) mock_opts.get_instance.return_value.get_all.return_value = { - "download_archive": "/tmp/a.txt", + "download_archive": file, "skip_download": True, } mock_read.return_value = ["arch"] @@ -128,11 +135,11 @@ class TestItemDTO: assert dto.download_skipped is False @patch("app.library.ItemDTO.archive_read") - def test_serialize_archives_finished(self, mock_read): + def test_serialize_archives_finished(self, mock_read, tmp_path: Path): # Given a finished item with archive info dto = ItemDTO(id="vid", title="t", url="u", folder="f") dto.archive_id = "arch" - dto._archive_file = "/tmp/a.txt" + dto._archive_file = _archive_path(tmp_path) dto.status = "finished" mock_read.return_value = ["arch"] @@ -186,7 +193,7 @@ class TestItemDTO: # Set up to allow add dto.archive_id = "arch" - dto._archive_file = "/tmp/a.txt" + dto._archive_file = "var/tests/archive.txt" dto.is_archivable = True dto.is_archived = False diff --git a/app/tests/test_nfo_maker.py b/app/tests/test_nfo_maker.py index 24d6cbd0..9ca37503 100644 --- a/app/tests/test_nfo_maker.py +++ b/app/tests/test_nfo_maker.py @@ -4,18 +4,8 @@ from pathlib import Path from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP -TEST_DIR = Path("var/tmp/tests_nfo") -TEST_DIR.mkdir(parents=True, exist_ok=True) - -def _clean_file(path: Path) -> None: - try: - path.unlink() - except FileNotFoundError: - pass - - -def sample_info_tv() -> dict: +def sample_info_tv(path: Path) -> dict: return { "id": "abc123", "title": "Test Show - S01E02", @@ -25,11 +15,11 @@ def sample_info_tv() -> dict: "episode_number": 2, "description": "First line.\n#tag\n00:01:23 Intro", "extractor": "youtube", - "filename": str(TEST_DIR / "test_show_s01e02.mkv"), + "filename": str(path), } -def sample_info_movie() -> dict: +def sample_info_movie(path: Path) -> dict: return { "id": "mov001", "title": "Test Movie", @@ -38,60 +28,46 @@ def sample_info_movie() -> dict: "duration": 7200, "thumbnail": "http://example.com/thumb.jpg", "webpage_url": "http://example.com/trailer", - "filename": str(TEST_DIR / "test_movie.mp4"), + "filename": str(path), } -def test_generate_nfo_tv_mode(tmp_path): - # arrange - TEST_DIR.mkdir(parents=True, exist_ok=True) - info = sample_info_tv() - media_file = Path(info["filename"]) - media_file.write_text("dummy") +def test_generate_nfo_tv_mode(tmp_path: Path) -> None: + media_file = tmp_path / "test_show_s01e02.mkv" + info = sample_info_tv(media_file) + media_file.write_text("dummy", encoding="utf-8") nfo_path = media_file.with_suffix(".nfo") - _clean_file(nfo_path) + nfo_path.unlink(missing_ok=True) - # act res = NFOMakerPP.generate_nfo(info_dict=info, filepath=media_file, mode="tv", overwrite=True, prefix=True) - # assert assert res["success"] is True assert nfo_path.exists() content = nfo_path.read_text(encoding="utf-8") assert "" in content assert "Test Show - S01E02" in content - # description cleaned (no timestamps / hashtags) assert "Intro" not in content assert "#tag" not in content - # also ensure PostProcessor.run wrapper produces the same result and syncs mtime - _clean_file(nfo_path) + nfo_path.unlink() pp = NFOMakerPP(None, mode="tv", prefix=True) media_mtime = media_file.stat().st_mtime - _, info_out = pp.run(info.copy()) + pp.run(info.copy()) assert nfo_path.exists() nfo_mtime = nfo_path.stat().st_mtime assert abs(nfo_mtime - media_mtime) < 2.0 - # cleanup - media_file.unlink() - nfo_path.unlink() - -def test_generate_nfo_movie_mode_and_run_wrapper(tmp_path): - # arrange - TEST_DIR.mkdir(parents=True, exist_ok=True) - info = sample_info_movie() - media_file = Path(info["filename"]) - media_file.write_text("dummy-movie") +def test_generate_nfo_movie_mode_and_run_wrapper(tmp_path: Path) -> None: + media_file = tmp_path / "test_movie.mp4" + info = sample_info_movie(media_file) + media_file.write_text("dummy-movie", encoding="utf-8") nfo_path = media_file.with_suffix(".nfo") - _clean_file(nfo_path) + nfo_path.unlink(missing_ok=True) - # act: generate_nfo static res = NFOMakerPP.generate_nfo(info_dict=info, filepath=media_file, mode="movie", overwrite=True, prefix=False) - # assert static helper assert res["success"] is True assert nfo_path.exists() content = nfo_path.read_text(encoding="utf-8") @@ -99,26 +75,16 @@ def test_generate_nfo_movie_mode_and_run_wrapper(tmp_path): assert "Test Movie" in content assert "7200" in content or "7200" in content - # cleanup nfo and re-run via PP.run wrapper nfo_path.unlink() - # call through instance run() to ensure wrapper works (mtime sync path exercised) pp = NFOMakerPP(None, mode="movie", prefix=False) - # ensure file exists assert media_file.exists() - # capture media file mtime before run media_mtime = media_file.stat().st_mtime - _, info_out = pp.run(info.copy()) + pp.run(info.copy()) - # run() should not raise and should produce nfo file assert nfo_path.exists() - # verify mtime was synced (allow small rounding differences) nfo_mtime = nfo_path.stat().st_mtime assert abs(nfo_mtime - media_mtime) < 2.0 - - # cleanup - media_file.unlink() - nfo_path.unlink() From 34e99951fc8d8e7f54f08debdd5c7a4234f533c4 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Wed, 13 May 2026 06:58:43 +0300 Subject: [PATCH 4/5] fix: tasks handler summary should reflect correct status. --- app/features/tasks/definitions/service.py | 29 ++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/app/features/tasks/definitions/service.py b/app/features/tasks/definitions/service.py index eca88441..5e2aa50a 100644 --- a/app/features/tasks/definitions/service.py +++ b/app/features/tasks/definitions/service.py @@ -70,6 +70,7 @@ class TaskHandle: s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []} handler_groups: dict[str, list[tuple[HandleTask, type]]] = {} + dispatches: list[tuple[HandleTask, type, asyncio.Task[TaskResult | TaskFailure | None]]] = [] tasks: list[TaskModel] = await self._repo.list() @@ -95,7 +96,6 @@ class TaskHandle: if handler_name not in handler_groups: handler_groups[handler_name] = [] handler_groups[handler_name].append((task, handler)) - s["h"].append(task.name) except Exception as e: LOG.error(f"Failed to handle task '{task.name}'. '{e!s}'.") s["f"].append(task.name) @@ -112,8 +112,27 @@ class TaskHandle: name=f"taskHandler-{task.id}", ) t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t)) + dispatches.append((task, handler, t)) except Exception as e: LOG.error(f"Failed to dispatch task '{task.name}'. '{e!s}'.") + s["f"].append(task.name) + + if dispatches: + results = await asyncio.gather(*(t for _, _, t in dispatches), return_exceptions=True) + + for (task, handler, _), result in zip(dispatches, results, strict=True): + if isinstance(result, TaskResult): + s["h"].append(task.name) + continue + + if isinstance(result, TaskFailure): + s["f"].append(task.name) + continue + + if result is None: + LOG.error(f"Handler '{handler.__name__}' returned no result for task '{task.name}'.") + + s["f"].append(task.name) if len(tasks) > 0: LOG.info( @@ -193,7 +212,11 @@ class TaskHandle: raise if isinstance(extraction, TaskFailure): - LOG.error(f"Handler '{handler.__name__}' failed to extract items: {extraction.message}") + msg: str = extraction.message + if extraction.error and extraction.error != extraction.message: + msg = f"{msg} {extraction.error}" + + LOG.error(f"Handler '{handler.__name__}' failed to extract items for task '{task.name}': {msg}") return extraction if not isinstance(extraction, TaskResult): @@ -226,7 +249,7 @@ class TaskHandle: for item in raw_items: if not isinstance(item, TaskItem): - LOG.warning("Handler '{handler.__name__}' produced non-TaskItem entry: {item!r}") + LOG.warning(f"Handler '{handler.__name__}' returned unexpected result: {item!r}") continue url: str = item.url From 1d52ea61da1065c46abff58afe0a587fcc0540ab Mon Sep 17 00:00:00 2001 From: arabcoders Date: Wed, 13 May 2026 08:39:22 +0300 Subject: [PATCH 5/5] refactor: do not log httpx related as exceptions but errors. --- app/features/tasks/definitions/handlers/generic.py | 4 +++- app/features/tasks/definitions/handlers/rss.py | 4 ++++ app/features/tasks/definitions/handlers/tver.py | 4 ++++ app/features/tasks/definitions/handlers/twitch.py | 4 ++++ app/features/tasks/definitions/handlers/youtube.py | 4 ++++ 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/app/features/tasks/definitions/handlers/generic.py b/app/features/tasks/definitions/handlers/generic.py index 5cf21ce1..6369bfb7 100644 --- a/app/features/tasks/definitions/handlers/generic.py +++ b/app/features/tasks/definitions/handlers/generic.py @@ -12,6 +12,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any from urllib.parse import urljoin +import httpx import jmespath from parsel import Selector @@ -31,7 +32,6 @@ from ._base_handler import BaseHandler if TYPE_CHECKING: from pathlib import Path - import httpx from parsel.selector import SelectorList LOG: logging.Logger = logging.getLogger("handlers.generic") @@ -140,6 +140,8 @@ class GenericTaskHandler(BaseHandler): body_text, json_data = await GenericTaskHandler._fetch_content( url=target_url, definition=definition, ytdlp_opts=ytdlp_opts ) + except httpx.HTTPError as exc: + return TaskFailure(message="Failed to fetch target URL.", error=str(exc)) except Exception as exc: LOG.exception(exc) return TaskFailure(message="Failed to fetch target URL.", error=str(exc)) diff --git a/app/features/tasks/definitions/handlers/rss.py b/app/features/tasks/definitions/handlers/rss.py index 0e4f7b12..964afbef 100644 --- a/app/features/tasks/definitions/handlers/rss.py +++ b/app/features/tasks/definitions/handlers/rss.py @@ -4,6 +4,8 @@ import re from typing import TYPE_CHECKING, Any from xml.etree.ElementTree import Element +import httpx + from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.ytdlp.extractor import fetch_info from app.features.ytdlp.utils import get_archive_id @@ -151,6 +153,8 @@ class RssGenericHandler(BaseHandler): try: feed_url, items, real_count = await RssGenericHandler._get(task, params, parsed) + except httpx.HTTPError as exc: + return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc)) except Exception as exc: LOG.exception(exc) return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc)) diff --git a/app/features/tasks/definitions/handlers/tver.py b/app/features/tasks/definitions/handlers/tver.py index d2e5e19d..ff413e00 100644 --- a/app/features/tasks/definitions/handlers/tver.py +++ b/app/features/tasks/definitions/handlers/tver.py @@ -1,6 +1,8 @@ import logging import re +import httpx + from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.ytdlp.utils import get_archive_id @@ -151,6 +153,8 @@ class TverHandler(BaseHandler): try: feed_url, items, has_items = await TverHandler._collect_feed(task, params, series_id) + except httpx.HTTPError as exc: + return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc)) except Exception as exc: LOG.exception(exc) return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc)) diff --git a/app/features/tasks/definitions/handlers/twitch.py b/app/features/tasks/definitions/handlers/twitch.py index 2cb9c43d..8aac9b29 100644 --- a/app/features/tasks/definitions/handlers/twitch.py +++ b/app/features/tasks/definitions/handlers/twitch.py @@ -3,6 +3,8 @@ import re from typing import TYPE_CHECKING from xml.etree.ElementTree import Element +import httpx + from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.ytdlp.utils import get_archive_id @@ -83,6 +85,8 @@ class TwitchHandler(BaseHandler): try: feed_url, items, has_items = await TwitchHandler._collect_feed(task, params, handle_name) + except httpx.HTTPError as exc: + return TaskFailure(message="Failed to fetch Twitch feed.", error=str(exc)) except Exception as exc: LOG.exception(exc) return TaskFailure(message="Failed to fetch Twitch feed.", error=str(exc)) diff --git a/app/features/tasks/definitions/handlers/youtube.py b/app/features/tasks/definitions/handlers/youtube.py index 1fb9d576..777f0c5e 100644 --- a/app/features/tasks/definitions/handlers/youtube.py +++ b/app/features/tasks/definitions/handlers/youtube.py @@ -3,6 +3,8 @@ import re from typing import TYPE_CHECKING, Any from xml.etree.ElementTree import Element +import httpx + from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.ytdlp.utils import get_archive_id @@ -98,6 +100,8 @@ class YoutubeHandler(BaseHandler): try: feed_url, items, real_count = await YoutubeHandler._get(task, params, parsed) + except httpx.HTTPError as exc: + return TaskFailure(message="Failed to fetch YouTube feed.", error=str(exc)) except Exception as exc: LOG.exception(exc) return TaskFailure(message="Failed to fetch YouTube feed.", error=str(exc))