[FIX] Force generic handler to generate real archive_id for generic urls.

app/library/Utils.py

@@ -251,7 +251,7 @@ def extract_info(
     no_archive: bool = False,
     follow_redirect: bool = False,
     sanitize_info: bool = False,
-    **kwargs,  # noqa: ARG001
+    **kwargs,
 ) -> dict:
     """
     Extracts video information from the given URL.
@@ -315,6 +315,11 @@ def extract_info(
 
         params["logger"] = log_wrapper
 
+    if kwargs.get("no_log", False):
+        params["logger"] = LogWrapper()
+        params["quiet"] = True
+        params["no_warnings"] = True
+
     if no_archive and "download_archive" in params:
         del params["download_archive"]
 

app/library/task_handlers/generic.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import asyncio
 import fnmatch
+import hashlib
 import json
 import logging
 import re
@@ -19,9 +20,10 @@ from parsel import Selector
 from parsel.selector import SelectorList
 from yt_dlp.utils.networking import random_user_agent
 
+from app.library.cache import Cache
 from app.library.config import Config
 from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult
-from app.library.Utils import get_archive_id
+from app.library.Utils import extract_info, get_archive_id
 
 from ._base_handler import BaseHandler
 
@@ -29,7 +31,7 @@ if TYPE_CHECKING:
     from parsel.selector import SelectorList
 
 LOG: logging.Logger = logging.getLogger(__name__)
-
+CACHE: Cache = Cache()
 
 @dataclass(slots=True)
 class MatchRule:
@@ -700,11 +702,35 @@ class GenericTaskHandler(BaseHandler):
             idDict: str | None = get_archive_id(url=url)
             archive_id: str | None = idDict.get("archive_id")
             if not archive_id:
-                LOG.warning(
+                cache_key: str = hashlib.sha256(f"{task.name}-{url}".encode()).hexdigest()
-                    f"[{definition.name}]: '{task.name}': Could not compute archive ID for video '{url}' in feed. generating one."
+                if CACHE.has(cache_key):
-                )
+                    archive_id = CACHE.get(cache_key)
+                    if not archive_id:
+                        continue
+                else:
+                    LOG.warning(
+                        f"[{definition.name}]: '{task.name}': Unable to generate static archive id for '{url}' in feed. Doing real request to fetch yt-dlp archive id."
+                    )
 
-                archive_id = f"generic {_generic_id(url)}"
+                    info = extract_info(
+                        config=task.get_ytdlp_opts().get_all(),
+                        url=url,
+                        no_archive=True,
+                        no_log=True,
+                    )
+
+                    if not info:
+                        LOG.error(f"[{definition.name}]: '{task.name}': Failed to extract info for URL '{url}' to generate archive ID. Skipping.")
+                        CACHE.set(cache_key, None)
+                        continue
+
+                    if not info.get("id") or not info.get("extractor_key"):
+                        LOG.error(f"[{definition.name}]: '{task.name}': Incomplete info extracted for URL '{url}' to generate archive ID. Skipping.")
+                        CACHE.set(cache_key, None)
+                        continue
+
+                    archive_id = f"{str(info.get('extractor_key', '')).lower()} {info.get('id')}"
+                    CACHE.set(cache_key, archive_id)
 
             metadata: dict[str, str] = {
                 k: v for k, v in entry.items() if k not in {"link", "url", "title", "published", "archive_id"}

app/routes/api/_static.py

@@ -25,6 +25,7 @@ FRONTEND_ROUTES: list[str] = [
     "/console/",
     "/presets/",
     "/tasks/",
+    "/task_definitions/",
     "/notifications/",
     "/changelog/",
     "/logs/",

app/routes/api/yt_dlp.py

@@ -244,7 +244,7 @@ async def get_options() -> Response:
 @route("POST", "api/yt-dlp/archive_id/", "get_archive_ids")
 async def get_archive_ids(request: Request, config: Config) -> Response:
     """
-    Get the yt-dlp CLI options.
+    Get the archive IDs for the given URLs.
 
     Returns:
         Response: The response object with the yt-dlp CLI options.

app/tests/test_generic_task_handler.py

@@ -1,6 +1,7 @@
 import json
 from pathlib import Path
 from types import SimpleNamespace
+from unittest.mock import patch
 
 import pytest
 
@@ -308,14 +309,19 @@ async def test_generic_task_handler_inspect(monkeypatch):
 
     monkeypatch.setattr(GenericTaskHandler, "_fetch_content", staticmethod(fake_fetch_content))
 
-    task = Task(id="inspect", name="Inspect", url="https://example.com/api")
+    # Mock extract_info to return valid info with required fields for archive ID generation
-    result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task)
+    def fake_extract_info(config, url, **kwargs):  # noqa: ARG001
+        return {"id": "test_video_1", "extractor_key": "Example"}
 
-    assert isinstance(result, TaskResult)
+    with patch("app.library.task_handlers.generic.extract_info", side_effect=fake_extract_info):
-    assert len(result.items) == 1
+        task = Task(id="inspect", name="Inspect", url="https://example.com/api")
-    item = result.items[0]
+        result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task)
-    assert item.url == "https://example.com/video/1"
+
-    assert item.title == "First"
+        assert isinstance(result, TaskResult)
+        assert len(result.items) == 1
+        item = result.items[0]
+        assert item.url == "https://example.com/video/1"
+        assert item.title == "First"
 
 
 def test_parse_items_handles_json_top_level_list():