From 13b13d1f7bdfb5b376d93fae64b6ade64e03d91d Mon Sep 17 00:00:00 2001 From: arabcoders Date: Tue, 21 Oct 2025 23:58:01 +0300 Subject: [PATCH] [FIX] Force generic handler to generate real archive_id for generic urls. --- app/library/Utils.py | 7 ++++- app/library/task_handlers/generic.py | 38 ++++++++++++++++++++++---- app/routes/api/_static.py | 1 + app/routes/api/yt_dlp.py | 2 +- app/tests/test_generic_task_handler.py | 20 +++++++++----- 5 files changed, 53 insertions(+), 15 deletions(-) diff --git a/app/library/Utils.py b/app/library/Utils.py index 3b327c93..1c34e34a 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -251,7 +251,7 @@ def extract_info( no_archive: bool = False, follow_redirect: bool = False, sanitize_info: bool = False, - **kwargs, # noqa: ARG001 + **kwargs, ) -> dict: """ Extracts video information from the given URL. @@ -315,6 +315,11 @@ def extract_info( params["logger"] = log_wrapper + if kwargs.get("no_log", False): + params["logger"] = LogWrapper() + params["quiet"] = True + params["no_warnings"] = True + if no_archive and "download_archive" in params: del params["download_archive"] diff --git a/app/library/task_handlers/generic.py b/app/library/task_handlers/generic.py index 6cde2b53..25cc2f32 100644 --- a/app/library/task_handlers/generic.py +++ b/app/library/task_handlers/generic.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio import fnmatch +import hashlib import json import logging import re @@ -19,9 +20,10 @@ from parsel import Selector from parsel.selector import SelectorList from yt_dlp.utils.networking import random_user_agent +from app.library.cache import Cache from app.library.config import Config from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult -from app.library.Utils import get_archive_id +from app.library.Utils import extract_info, get_archive_id from ._base_handler import BaseHandler @@ -29,7 +31,7 @@ if TYPE_CHECKING: from parsel.selector import SelectorList LOG: logging.Logger = logging.getLogger(__name__) - +CACHE: Cache = Cache() @dataclass(slots=True) class MatchRule: @@ -700,11 +702,35 @@ class GenericTaskHandler(BaseHandler): idDict: str | None = get_archive_id(url=url) archive_id: str | None = idDict.get("archive_id") if not archive_id: - LOG.warning( - f"[{definition.name}]: '{task.name}': Could not compute archive ID for video '{url}' in feed. generating one." - ) + cache_key: str = hashlib.sha256(f"{task.name}-{url}".encode()).hexdigest() + if CACHE.has(cache_key): + archive_id = CACHE.get(cache_key) + if not archive_id: + continue + else: + LOG.warning( + f"[{definition.name}]: '{task.name}': Unable to generate static archive id for '{url}' in feed. Doing real request to fetch yt-dlp archive id." + ) - archive_id = f"generic {_generic_id(url)}" + info = extract_info( + config=task.get_ytdlp_opts().get_all(), + url=url, + no_archive=True, + no_log=True, + ) + + if not info: + LOG.error(f"[{definition.name}]: '{task.name}': Failed to extract info for URL '{url}' to generate archive ID. Skipping.") + CACHE.set(cache_key, None) + continue + + if not info.get("id") or not info.get("extractor_key"): + LOG.error(f"[{definition.name}]: '{task.name}': Incomplete info extracted for URL '{url}' to generate archive ID. Skipping.") + CACHE.set(cache_key, None) + continue + + archive_id = f"{str(info.get('extractor_key', '')).lower()} {info.get('id')}" + CACHE.set(cache_key, archive_id) metadata: dict[str, str] = { k: v for k, v in entry.items() if k not in {"link", "url", "title", "published", "archive_id"} diff --git a/app/routes/api/_static.py b/app/routes/api/_static.py index d1f40ca5..f00faa1a 100644 --- a/app/routes/api/_static.py +++ b/app/routes/api/_static.py @@ -25,6 +25,7 @@ FRONTEND_ROUTES: list[str] = [ "/console/", "/presets/", "/tasks/", + "/task_definitions/", "/notifications/", "/changelog/", "/logs/", diff --git a/app/routes/api/yt_dlp.py b/app/routes/api/yt_dlp.py index 72144e92..873be860 100644 --- a/app/routes/api/yt_dlp.py +++ b/app/routes/api/yt_dlp.py @@ -244,7 +244,7 @@ async def get_options() -> Response: @route("POST", "api/yt-dlp/archive_id/", "get_archive_ids") async def get_archive_ids(request: Request, config: Config) -> Response: """ - Get the yt-dlp CLI options. + Get the archive IDs for the given URLs. Returns: Response: The response object with the yt-dlp CLI options. diff --git a/app/tests/test_generic_task_handler.py b/app/tests/test_generic_task_handler.py index 6654545e..49cad7f3 100644 --- a/app/tests/test_generic_task_handler.py +++ b/app/tests/test_generic_task_handler.py @@ -1,6 +1,7 @@ import json from pathlib import Path from types import SimpleNamespace +from unittest.mock import patch import pytest @@ -308,14 +309,19 @@ async def test_generic_task_handler_inspect(monkeypatch): monkeypatch.setattr(GenericTaskHandler, "_fetch_content", staticmethod(fake_fetch_content)) - task = Task(id="inspect", name="Inspect", url="https://example.com/api") - result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task) + # Mock extract_info to return valid info with required fields for archive ID generation + def fake_extract_info(config, url, **kwargs): # noqa: ARG001 + return {"id": "test_video_1", "extractor_key": "Example"} - assert isinstance(result, TaskResult) - assert len(result.items) == 1 - item = result.items[0] - assert item.url == "https://example.com/video/1" - assert item.title == "First" + with patch("app.library.task_handlers.generic.extract_info", side_effect=fake_extract_info): + task = Task(id="inspect", name="Inspect", url="https://example.com/api") + result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task) + + assert isinstance(result, TaskResult) + assert len(result.items) == 1 + item = result.items[0] + assert item.url == "https://example.com/video/1" + assert item.title == "First" def test_parse_items_handles_json_top_level_list():