Merge pull request #485 from arabcoders/dev

Refactor: refactor postprocessor into correct directory

app/library/Utils.py

@@ -98,6 +98,52 @@ class FileLogFormatter(logging.Formatter):
         return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
 
 
+def patch_metadataparser() -> None:
+    """
+    Patches yt_dlp MetadataParserPP action to handle subprocess pickling issues.
+    """
+    try:
+        from yt_dlp.postprocessor.metadataparser import MetadataParserPP
+        from yt_dlp.utils import Namespace
+    except Exception as exc:
+        LOG.warning(f"Unable to import yt_dlp metadata parser for patching: {exc!s}")
+        return
+
+    if getattr(MetadataParserPP.Actions, "_ytptube_patched", False):
+        return
+
+    class _ActionNS(Namespace):
+        _ACTIONS_STR: list[str] = []
+
+        @staticmethod
+        def _get_name(func) -> str | None:
+            if not callable(func):
+                return None
+
+            target = getattr(func, "__func__", func)
+            module_name = getattr(target, "__module__", None)
+            qual_name = getattr(target, "__qualname__", getattr(target, "__name__", None))
+
+            return f"{module_name}.{qual_name}" if module_name and qual_name else None
+
+        def __contains__(self, candidate: object) -> bool:
+            if candidate in self.__dict__.values():
+                return True
+
+            if func_name := _ActionNS._get_name(candidate):
+                if len(_ActionNS._ACTIONS_STR) < 1:
+                    _ActionNS._ACTIONS_STR.extend([_ActionNS._get_name(value) for value in self.__dict__.values()])
+
+                return func_name in _ActionNS._ACTIONS_STR
+
+            return False
+
+    actions_dict: dict[str, Any] = dict(MetadataParserPP.Actions.items_)
+    MetadataParserPP.Actions = _ActionNS(**actions_dict)
+    MetadataParserPP.Actions._ytptube_patched = True
+    LOG.debug("MetadataParserPP action namespace patch applied successfully.")
+
+
 def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
     """
     Decorator that applies an LRU cache with a time-to-live (TTL) to a function.
@@ -591,9 +637,7 @@ def arg_converter(
             yt_dlp.options.create_parser = create_parser
 
     try:
-        from app.postprocessors.patch_metadata_parser import ensure_patch
+        patch_metadataparser()
-
-        ensure_patch()
     except Exception as exc:
         LOG.debug("Metadata parser patch failed to apply: %s", exc)
 

app/library/ytdlp.py

@@ -4,8 +4,6 @@ from typing import Any
 import yt_dlp
 from yt_dlp.utils import make_archive_id
 
-import app.postprocessors
-
 
 class _ArchiveProxy:
     """

app/postprocessors/__init__.py

@@ -1,15 +0,0 @@
-from yt_dlp.globals import postprocessors
-
-from .nfo_maker import NFOMakerPP
-
-_ytptube_pps: list[str] = {"NFOMakerPP": NFOMakerPP}
-
-
-postprocessors.value.update(_ytptube_pps)
-
-__all__: list = list(_ytptube_pps.values())
-
-# Apply patch to fix MetadataParserPP assertion error
-from .patch_metadata_parser import ensure_patch
-
-ensure_patch()

app/postprocessors/patch_metadata_parser.py

@@ -1,60 +0,0 @@
-"""
-Patches yt_dlp.postprocessor.metadataparser.MetadataParserPP action namespace to handle duplicated class objects.
-
-This patch is necessary due to to how we parse yt-dlp cli options. on top fo that we have pickling issue as well.
-So we need to compare callables structurally rather than by identity as the identity may differ across instances.
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import Any
-
-LOG: logging.Logger = logging.getLogger(__name__)
-
-
-def _callable_fingerprint(func: object) -> tuple[Any, ...] | None:
-    if not callable(func):
-        return None
-
-    target = getattr(func, "__func__", func)
-    code = getattr(target, "__code__", None)
-    defaults = getattr(target, "__defaults__", None)
-    kwdefaults = getattr(target, "__kwdefaults__", None)
-
-    return (
-        getattr(target, "__module__", None),
-        getattr(target, "__qualname__", getattr(target, "__name__", None)),
-        getattr(code, "co_code", None),
-        getattr(code, "co_consts", None),
-        defaults,
-        kwdefaults,
-    )
-
-
-def ensure_patch() -> None:
-    try:
-        from yt_dlp.postprocessor.metadataparser import MetadataParserPP
-        from yt_dlp.utils import Namespace
-    except Exception as exc:
-        LOG.warning(f"Unable to import yt_dlp metadata parser for patching: {exc!s}")
-        return
-
-    if getattr(MetadataParserPP.Actions, "_ytptube_patched", False):
-        return
-
-    class _ActionNamespace(Namespace):
-        def __contains__(self, candidate: object) -> bool:
-            if candidate in self.__dict__.values():
-                return True
-
-            candidate_fp = _callable_fingerprint(candidate)
-            if candidate_fp is None:
-                return False
-
-            return any(_callable_fingerprint(value) == candidate_fp for value in self.__dict__.values())
-
-    actions_dict = dict(MetadataParserPP.Actions.items_)
-    MetadataParserPP.Actions = _ActionNamespace(**actions_dict)
-    MetadataParserPP.Actions._ytptube_patched = True
-    LOG.debug("MetadataParserPP action namespace patch applied successfully.")

app/routes/api/tasks.py

@@ -1,6 +1,6 @@
 import logging
 import uuid
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from aiohttp import web
 from aiohttp.web import Request, Response
@@ -12,6 +12,9 @@ from app.library.router import route
 from app.library.Tasks import Task, TaskFailure, TaskResult, Tasks
 from app.library.Utils import get_channel_images, get_file, init_class, validate_url, validate_uuid
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 LOG: logging.Logger = logging.getLogger(__name__)
 
 
@@ -255,10 +258,10 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R
 
         from yt_dlp.utils import sanitize_filename
 
-        from app.postprocessors.nfo_maker import NFOMakerPP
+        from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP
 
-        title = sanitize_filename(info.get("title"))
+        title: str = sanitize_filename(info.get("title"))
-        filename = save_path / f"{title} [{info.get('id')}].info.json"
+        filename: Path = save_path / f"{title} [{info.get('id')}].info.json"
         filename.write_text(encoder.encode(metadata), encoding="utf-8")
         info["json_file"] = str(filename.relative_to(config.download_path))