From 3da5e54774e22e9f5b38852979863a639aa72e72 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Wed, 12 Nov 2025 21:56:29 +0300 Subject: [PATCH] Refactor: move nfo maker into correct namespace and move metadataPP patcher into utils --- app/library/Utils.py | 50 +++++++++++++++- app/library/ytdlp.py | 2 - app/postprocessors/__init__.py | 15 ----- app/postprocessors/patch_metadata_parser.py | 60 ------------------- app/routes/api/tasks.py | 11 ++-- .../postprocessor}/nfo_maker.py | 0 6 files changed, 54 insertions(+), 84 deletions(-) delete mode 100644 app/postprocessors/__init__.py delete mode 100644 app/postprocessors/patch_metadata_parser.py rename app/{postprocessors => yt_dlp_plugins/postprocessor}/nfo_maker.py (100%) diff --git a/app/library/Utils.py b/app/library/Utils.py index 358c26a2..1313f99f 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -98,6 +98,52 @@ class FileLogFormatter(logging.Formatter): return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds") +def patch_metadataparser() -> None: + """ + Patches yt_dlp MetadataParserPP action to handle subprocess pickling issues. + """ + try: + from yt_dlp.postprocessor.metadataparser import MetadataParserPP + from yt_dlp.utils import Namespace + except Exception as exc: + LOG.warning(f"Unable to import yt_dlp metadata parser for patching: {exc!s}") + return + + if getattr(MetadataParserPP.Actions, "_ytptube_patched", False): + return + + class _ActionNS(Namespace): + _ACTIONS_STR: list[str] = [] + + @staticmethod + def _get_name(func) -> str | None: + if not callable(func): + return None + + target = getattr(func, "__func__", func) + module_name = getattr(target, "__module__", None) + qual_name = getattr(target, "__qualname__", getattr(target, "__name__", None)) + + return f"{module_name}.{qual_name}" if module_name and qual_name else None + + def __contains__(self, candidate: object) -> bool: + if candidate in self.__dict__.values(): + return True + + if func_name := _ActionNS._get_name(candidate): + if len(_ActionNS._ACTIONS_STR) < 1: + _ActionNS._ACTIONS_STR.extend([_ActionNS._get_name(value) for value in self.__dict__.values()]) + + return func_name in _ActionNS._ACTIONS_STR + + return False + + actions_dict: dict[str, Any] = dict(MetadataParserPP.Actions.items_) + MetadataParserPP.Actions = _ActionNS(**actions_dict) + MetadataParserPP.Actions._ytptube_patched = True + LOG.debug("MetadataParserPP action namespace patch applied successfully.") + + def timed_lru_cache(ttl_seconds: int, max_size: int = 128): """ Decorator that applies an LRU cache with a time-to-live (TTL) to a function. @@ -591,9 +637,7 @@ def arg_converter( yt_dlp.options.create_parser = create_parser try: - from app.postprocessors.patch_metadata_parser import ensure_patch - - ensure_patch() + patch_metadataparser() except Exception as exc: LOG.debug("Metadata parser patch failed to apply: %s", exc) diff --git a/app/library/ytdlp.py b/app/library/ytdlp.py index d52a97c8..01d5670f 100644 --- a/app/library/ytdlp.py +++ b/app/library/ytdlp.py @@ -4,8 +4,6 @@ from typing import Any import yt_dlp from yt_dlp.utils import make_archive_id -import app.postprocessors - class _ArchiveProxy: """ diff --git a/app/postprocessors/__init__.py b/app/postprocessors/__init__.py deleted file mode 100644 index d04c97c3..00000000 --- a/app/postprocessors/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from yt_dlp.globals import postprocessors - -from .nfo_maker import NFOMakerPP - -_ytptube_pps: list[str] = {"NFOMakerPP": NFOMakerPP} - - -postprocessors.value.update(_ytptube_pps) - -__all__: list = list(_ytptube_pps.values()) - -# Apply patch to fix MetadataParserPP assertion error -from .patch_metadata_parser import ensure_patch - -ensure_patch() diff --git a/app/postprocessors/patch_metadata_parser.py b/app/postprocessors/patch_metadata_parser.py deleted file mode 100644 index 05a59537..00000000 --- a/app/postprocessors/patch_metadata_parser.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Patches yt_dlp.postprocessor.metadataparser.MetadataParserPP action namespace to handle duplicated class objects. - -This patch is necessary due to to how we parse yt-dlp cli options. on top fo that we have pickling issue as well. -So we need to compare callables structurally rather than by identity as the identity may differ across instances. -""" - -from __future__ import annotations - -import logging -from typing import Any - -LOG: logging.Logger = logging.getLogger(__name__) - - -def _callable_fingerprint(func: object) -> tuple[Any, ...] | None: - if not callable(func): - return None - - target = getattr(func, "__func__", func) - code = getattr(target, "__code__", None) - defaults = getattr(target, "__defaults__", None) - kwdefaults = getattr(target, "__kwdefaults__", None) - - return ( - getattr(target, "__module__", None), - getattr(target, "__qualname__", getattr(target, "__name__", None)), - getattr(code, "co_code", None), - getattr(code, "co_consts", None), - defaults, - kwdefaults, - ) - - -def ensure_patch() -> None: - try: - from yt_dlp.postprocessor.metadataparser import MetadataParserPP - from yt_dlp.utils import Namespace - except Exception as exc: - LOG.warning(f"Unable to import yt_dlp metadata parser for patching: {exc!s}") - return - - if getattr(MetadataParserPP.Actions, "_ytptube_patched", False): - return - - class _ActionNamespace(Namespace): - def __contains__(self, candidate: object) -> bool: - if candidate in self.__dict__.values(): - return True - - candidate_fp = _callable_fingerprint(candidate) - if candidate_fp is None: - return False - - return any(_callable_fingerprint(value) == candidate_fp for value in self.__dict__.values()) - - actions_dict = dict(MetadataParserPP.Actions.items_) - MetadataParserPP.Actions = _ActionNamespace(**actions_dict) - MetadataParserPP.Actions._ytptube_patched = True - LOG.debug("MetadataParserPP action namespace patch applied successfully.") diff --git a/app/routes/api/tasks.py b/app/routes/api/tasks.py index 35c28fb8..7a01054e 100644 --- a/app/routes/api/tasks.py +++ b/app/routes/api/tasks.py @@ -1,6 +1,6 @@ import logging import uuid -from typing import Any +from typing import TYPE_CHECKING, Any from aiohttp import web from aiohttp.web import Request, Response @@ -12,6 +12,9 @@ from app.library.router import route from app.library.Tasks import Task, TaskFailure, TaskResult, Tasks from app.library.Utils import get_channel_images, get_file, init_class, validate_url, validate_uuid +if TYPE_CHECKING: + from pathlib import Path + LOG: logging.Logger = logging.getLogger(__name__) @@ -255,10 +258,10 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R from yt_dlp.utils import sanitize_filename - from app.postprocessors.nfo_maker import NFOMakerPP + from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP - title = sanitize_filename(info.get("title")) - filename = save_path / f"{title} [{info.get('id')}].info.json" + title: str = sanitize_filename(info.get("title")) + filename: Path = save_path / f"{title} [{info.get('id')}].info.json" filename.write_text(encoder.encode(metadata), encoding="utf-8") info["json_file"] = str(filename.relative_to(config.download_path)) diff --git a/app/postprocessors/nfo_maker.py b/app/yt_dlp_plugins/postprocessor/nfo_maker.py similarity index 100% rename from app/postprocessors/nfo_maker.py rename to app/yt_dlp_plugins/postprocessor/nfo_maker.py