From e085e680ccbbfbcc5324cd8e1c3f4b55b3976bb8 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Wed, 15 Oct 2025 18:55:50 +0300 Subject: [PATCH] [FEAT] create channel metadata and thumbnails. Closes #446 --- .vscode/settings.json | 1 + API.md | 53 ++++++++++++ app/library/Tasks.py | 59 +++++++++++++ app/library/Utils.py | 57 +++++++++++++ app/postprocessors/nfo_maker.py | 43 ++++++---- app/routes/api/tasks.py | 146 +++++++++++++++++++++++++++++++- ui/app/pages/tasks.vue | 68 +++++++++++++++ ui/app/utils/index.ts | 14 ++- ui/tests/utils/index.test.ts | 59 +++++++++++++ 9 files changed, 474 insertions(+), 26 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 77526555..d5aadcdc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -87,6 +87,7 @@ "jmespath", "jsonschema", "kibibytes", + "Kodi", "lastgroup", "levelno", "libcurl", diff --git a/API.md b/API.md index b9894a86..8bb25600 100644 --- a/API.md +++ b/API.md @@ -35,6 +35,7 @@ This document describes the available endpoints and their usage. All endpoints r - [POST /api/tasks/inspect](#post-apitasksinspect) - [POST /api/tasks/{id}/mark](#post-apitasksidmark) - [DELETE /api/tasks/{id}/mark](#delete-apitasksidmark) + - [POST /api/tasks/{id}/metadata](#post-apitasksidmetadata) - [GET /api/task\_definitions/](#get-apitask_definitions) - [GET /api/task\_definitions/{identifier}](#get-apitask_definitionsidentifier) - [POST /api/task\_definitions/](#post-apitask_definitions) @@ -658,6 +659,58 @@ or --- +### POST /api/tasks/{id}/metadata +**Purpose**: Generate metadata files for a scheduled task, including NFO files, thumbnails, and JSON metadata compatible with media centers. + +**Path Parameter**: +- `id`: Task ID. + +**Response**: +```json +{ + "id": "channel_or_playlist_id", + "id_type": "youtube|twitch|...", + "title": "Channel/Playlist Title", + "description": "Description text...", + "uploader": "Uploader name", + "tags": ["tag1", "tag2"], + "year": 2024, + "json_file": "path/to/Title [id].info.json", + "nfo_file": "path/to/tvshow.nfo", + "thumbnails": { + "poster": "path/to/poster.jpg", + "fanart": "path/to/fanart.jpg", + "thumb": "path/to/thumb.jpg", + "banner": "path/to/banner.jpg", + "icon": "path/to/icon.jpg", + "landscape": "path/to/landscape.jpg" + } +} +``` +or +```json +{ "error": "..." } +``` + +**Files Generated**: +- `tvshow.nfo` - NFO file for media center compatibility (Kodi, Jellyfin, Emby, etc.) +- `Title [id].info.json` - yt-dlp metadata file. +- `poster.jpg`, `fanart.jpg`, `thumb.jpg`, `banner.jpg`, `icon.jpg`, `landscape.jpg` - Thumbnail images (if available from the source) + +**Notes**: +- This endpoint fetches metadata from the URL associated with the task +- Files are saved to the task's configured folder (or default download path) +- Existing files will be overwritten. +- Thumbnail images are only downloaded if available from the source. +- The NFO file follows the Kodi tvshow.nfo format + +**Status Codes**: +- `200 OK` - Metadata generated successfully +- `400 Bad Request` - Missing task ID, invalid folder path, or failed to fetch metadata +- `404 Not Found` - Task does not exist + +--- + ### GET /api/task_definitions/ **Purpose**: Retrieve all task definitions. diff --git a/app/library/Tasks.py b/app/library/Tasks.py index f8ed0237..d91beb58 100644 --- a/app/library/Tasks.py +++ b/app/library/Tasks.py @@ -46,9 +46,27 @@ class Task: return Encoder().encode(self.serialize()) def get(self, key: str, default: Any = None) -> Any: + """ + Get a value from the task by key. + + Args: + key (str): The key to get. + default (Any): The default value if the key is not found. + + Returns: + Any: The value of the key or the default value. + + """ return self.serialize().get(key, default) def get_ytdlp_opts(self) -> YTDLPOpts: + """ + Get the yt-dlp options for the task. + + Returns: + YTDLPOpts: The yt-dlp options. + + """ params: YTDLPOpts = YTDLPOpts.get_instance() if self.preset: @@ -60,6 +78,13 @@ class Task: return params def mark(self) -> tuple[bool, str]: + """ + Mark the task's items as downloaded in the archive file. + + Returns: + tuple[bool, str]: A tuple indicating success and a message. + + """ ret = self._mark_logic() if isinstance(ret, tuple): return ret @@ -73,6 +98,13 @@ class Task: return (True, f"Task '{self.name}' items marked as downloaded.") def unmark(self) -> tuple[bool, str]: + """ + Unmark the task's items from the archive file. + + Returns: + tuple[bool, str]: A tuple indicating success and a message. + + """ ret: tuple[bool, str] | set[tuple[Path, set[str]]] = self._mark_logic() if isinstance(ret, tuple): return ret @@ -85,6 +117,33 @@ class Task: return (True, f"Removed '{self.name}' items from archive file.") + def fetch_metadata(self, full: bool = False) -> tuple[dict[str, Any] | None, bool, str]: + """ + Fetch metadata for the task's URL. + + Args: + full (bool): Whether to fetch full metadata including all entries for playlists. + + Returns: + tuple[dict[str, Any]|None, bool, str]: A tuple containing the metadata (or None on failure), a boolean + indicating if the operation was successful, and a message. + + """ + if not self.url: + return ({}, False, "No URL found in task parameters.") + + params = self.get_ytdlp_opts() + if not full: + params.add_cli("-I0", from_user=False) + + params = params.get_all() + + ie_info: dict | None = extract_info(params, self.url, no_archive=True, follow_redirect=False, cache=True) + if not ie_info or not isinstance(ie_info, dict): + return ({}, False, "Failed to extract information from URL.") + + return (ie_info, True, "") + def _mark_logic(self) -> tuple[bool, str] | set[tuple[Path, set[str]]]: if not self.url: return (False, "No URL found in task parameters.") diff --git a/app/library/Utils.py b/app/library/Utils.py index b6c4a992..3b327c93 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -1572,3 +1572,60 @@ def archive_delete(file: str | Path, ids: list[str]) -> bool: from app.library.Archiver import Archiver return Archiver.get_instance().delete(file, ids) + + +def get_channel_images(thumbnails: list[dict]) -> dict: + """ + Extract channel images from a list of thumbnail dictionaries. + + Args: + thumbnails (list[dict]): List of thumbnail dictionaries with keys 'url', 'width', 'height', and 'id'. + + Returns: + dict: A dictionary with keys 'icon', 'landscape', 'poster', 'thumb', 'fanart', and 'banner' mapped to their respective URLs. + + """ + artwork = {} + + for t in thumbnails: + url = t.get("url") + if not url: + continue + + width = t.get("width") + height = t.get("height") + tid = t.get("id", "") + + if not width or not height: + if "avatar_uncropped" in tid: + artwork["icon"] = url + + elif "banner_uncropped" in tid: + artwork["landscape"] = url + + continue + + ratio = width / height + + if 0.55 <= ratio <= 0.75: # portrait → poster + artwork["poster"] = url + elif 0.9 <= ratio <= 1.1: # square → thumb + artwork.setdefault("thumb", url) + elif ratio >= 5: # very wide + if width >= 1920: + artwork["fanart"] = url + else: + artwork["banner"] = url + elif 1.6 <= ratio <= 1.8: # landscape + artwork["landscape"] = url + + if "fanart" not in artwork and "banner" in artwork: + artwork["fanart"] = artwork["banner"] + + if "banner" not in artwork and "fanart" in artwork: + artwork["banner"] = artwork["fanart"] + + if "poster" not in artwork and "thumb" in artwork: + artwork["poster"] = artwork["thumb"] # optional fallback + + return artwork diff --git a/app/postprocessors/nfo_maker.py b/app/postprocessors/nfo_maker.py index 70ab7ff5..ac345fec 100644 --- a/app/postprocessors/nfo_maker.py +++ b/app/postprocessors/nfo_maker.py @@ -211,7 +211,7 @@ class NFOMakerPP(PostProcessor): # collapse multiline descriptions if "description" == resolved_key and isinstance(resolved_val, str): - resolved_val = self._clean_description(resolved_val) + resolved_val = NFOMakerPP._clean_description(resolved_val) if resolved_val not in (None, ""): data[nfo_name] = resolved_val @@ -321,15 +321,9 @@ class NFOMakerPP(PostProcessor): repl (dict[str, Any]): Replacement dictionary. """ - from xml.sax.saxutils import escape - - # escape XML on a copy safe_repl: dict[str, Any] = {} for k, v in repl.items(): - if isinstance(v, str): - safe_repl[k] = escape(v) - else: - safe_repl[k] = v + safe_repl[k] = NFOMakerPP._escape_text(v) # replace placeholders rendered = text @@ -357,7 +351,24 @@ class NFOMakerPP(PostProcessor): except Exception as e: self.to_screen(f"Error writing NFO file: {e}") - def _clean_description(self, text: str) -> str: + @staticmethod + def _escape_text(text: Any) -> Any: + """ + Escape text for XML. + + Args: + text (str): Text to escape. + + Returns: + Any: Escaped text if input is str, else original value. + + """ + from xml.sax.saxutils import escape + + return escape(text) if isinstance(text, str) else text + + @staticmethod + def _clean_description(text: str) -> str: """ Strip links, chapters/timestamps, pure hashtags/mentions, and promo lines. Return a compact single-line summary suitable for NFO . @@ -374,20 +385,20 @@ class NFOMakerPP(PostProcessor): continue # remove markdown links, keep labels - ln = self._MD_LINK.sub(r"\1", ln) + ln = NFOMakerPP._MD_LINK.sub(r"\1", ln) # drop lines that are clearly noise - if self._TIME_LINE_PAT.match(ln): + if NFOMakerPP._TIME_LINE_PAT.match(ln): continue - if self._HASHTAGS_LINE.match(ln): + if NFOMakerPP._HASHTAGS_LINE.match(ln): continue - if self._MENTION_LINE.match(ln): + if NFOMakerPP._MENTION_LINE.match(ln): continue - if self._PROMO_LINE_PAT.search(ln): + if NFOMakerPP._PROMO_LINE_PAT.search(ln): continue # strip raw/bare urls and domains - ln = self._URL_PAT.sub("", ln) + ln = NFOMakerPP._URL_PAT.sub("", ln) # collapse leftover multiple spaces and stray separators ln = re.sub(r"\s{2,}", " ", ln) @@ -402,7 +413,7 @@ class NFOMakerPP(PostProcessor): # optional minimum signal: if too short, fall back to original first sentence without links if 8 > len(summary.split()): - fallback = self._URL_PAT.sub("", text) + fallback = NFOMakerPP._URL_PAT.sub("", text) fallback = re.sub(r"\s{2,}", " ", fallback).strip() if 8 <= len(fallback.split()): summary = fallback diff --git a/app/routes/api/tasks.py b/app/routes/api/tasks.py index 75e66d91..3499ccb1 100644 --- a/app/routes/api/tasks.py +++ b/app/routes/api/tasks.py @@ -1,14 +1,17 @@ import logging import uuid +from typing import Any from aiohttp import web from aiohttp.web import Request, Response +from app.library.ag_utils import ag from app.library.config import Config from app.library.encoder import Encoder from app.library.router import route from app.library.Tasks import Task, TaskFailure, TaskResult, Tasks -from app.library.Utils import init_class, validate_url, validate_uuid +from app.library.Utils import get_channel_images, get_file, init_class, validate_url, validate_uuid +from app.postprocessors.nfo_maker import NFOMakerPP LOG: logging.Logger = logging.getLogger(__name__) @@ -195,3 +198,144 @@ async def task_unmark(request: Request, encoder: Encoder) -> Response: return web.json_response(data={"message": _message}, status=web.HTTPOk.status_code, dumps=encoder.encode) except ValueError as e: return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + + +@route("POST", "api/tasks/{id}/metadata", "tasks_metadata") +async def task_metadata(request: Request, config: Config, encoder: Encoder) -> Response: + """ + Generate metadata for the task. + + Args: + request (Request): The request object. + config (Config): The config instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object + + """ + task_id: str = request.match_info.get("id", None) + + if not task_id: + return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) + + tasks: Tasks = Tasks.get_instance() + try: + task: Task | None = tasks.get(task_id) + if not task: + return web.json_response( + data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code + ) + + (save_path, _) = get_file(config.download_path, task.folder) + if not str(save_path or "").startswith(str(config.download_path)): + return web.json_response(data={"error": "Invalid task folder."}, status=web.HTTPBadRequest.status_code) + + if not save_path.exists(): + save_path.mkdir(parents=True, exist_ok=True) + + metadata, status, message = task.fetch_metadata(full=False) + if not status: + return web.json_response(data={"error": message}, status=web.HTTPBadRequest.status_code) + + info = { + "id": ag(metadata, ["id", "channel_id"]), + "id_type": metadata.get("extractor", "").split(":")[0].lower() if metadata.get("extractor") else None, + "title": ag(metadata, ["title", "fulltitle"]) or None, + "description": metadata.get("description", ""), + "uploader": metadata.get("uploader", ""), + "tags": metadata.get("tags", []), + "year": metadata.get("release_year"), + "thumbnails": get_channel_images(metadata.get("thumbnails", {})), + } + + if not info.get("title"): + return web.json_response( + data={"error": "Failed to get title from metadata."}, status=web.HTTPBadRequest.status_code + ) + + from yt_dlp.utils import sanitize_filename + + title = sanitize_filename(info.get("title")) + filename = save_path / f"{title} [{info.get('id')}].info.json" + filename.write_text(encoder.encode(metadata), encoding="utf-8") + info["json_file"] = str(filename.relative_to(config.download_path)) + + filename = save_path / "tvshow.nfo" + info["nfo_file"] = f"{save_path}/{filename}" + + xml_content = "\n" + xml_content += f" {NFOMakerPP._escape_text(info.get('title'))}\n" + if info.get("description"): + xml_content += ( + f" {NFOMakerPP._escape_text(NFOMakerPP._clean_description(info.get('description')))}\n" + ) + if info.get("id"): + xml_content += f" {NFOMakerPP._escape_text(info.get('id'))}\n" + if info.get("id_type") and info.get("id"): + xml_content += f' {NFOMakerPP._escape_text(info.get("id"))}\n' + if info.get("uploader"): + xml_content += f" {NFOMakerPP._escape_text(info.get('uploader'))}\n" + if info.get("tags", []): + for tag in info.get("tags", []): + xml_content += f" {NFOMakerPP._escape_text(tag)}\n" + if info.get("year"): + xml_content += f" {info.get('year')}\n" + xml_content += " Continuing\n" + xml_content += "\n" + + filename.write_text(xml_content, encoding="utf-8") + + try: + import httpx + from yt_dlp.utils.networking import random_user_agent + + ytdlp_args: dict = task.get_ytdlp_opts().get_all() + opts: dict[str, Any] = { + "headers": { + "User-Agent": request.headers.get("User-Agent", ytdlp_args.get("user_agent", random_user_agent())), + }, + } + if proxy := ytdlp_args.get("proxy"): + opts["proxy"] = proxy + + try: + from httpx_curl_cffi import AsyncCurlTransport, CurlOpt + + opts["transport"] = AsyncCurlTransport( + impersonate="chrome", + default_headers=True, + curl_options={CurlOpt.FRESH_CONNECT: True}, + ) + opts.pop("headers", None) + except Exception: + pass + + async with httpx.AsyncClient(**opts) as client: + for key in info.get("thumbnails", {}): + try: + url = info["thumbnails"][key] + LOG.info(f"Fetching thumbnail '{key}' from '{url}'") + if not url: + continue + + try: + validate_url(url, allow_internal=config.allow_internal_urls) + except ValueError: + LOG.warning(f"Invalid thumbnail url '{url}'") + continue + + resp = await client.request(method="GET", url=url, follow_redirects=True) + + filename = save_path / f"{key}.jpg" + filename.write_bytes(resp.content) + info["thumbnails"][key] = str(filename.relative_to(config.download_path)) + except Exception as e: + LOG.warning(f"Failed to fetch thumbnail '{key}' from '{url}'. '{e!s}'") + continue + except Exception as e: + LOG.warning(f"Failed to fetch thumbnails. '{e!s}'") + + return web.json_response(data=info, status=web.HTTPOk.status_code, dumps=encoder.encode) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) diff --git a/ui/app/pages/tasks.vue b/ui/app/pages/tasks.vue index bfc8cbd1..10842ac9 100644 --- a/ui/app/pages/tasks.vue +++ b/ui/app/pages/tasks.vue @@ -200,6 +200,13 @@ Run now + + + Generate metadata + + +
+ Inspect Handler @@ -243,6 +250,9 @@ {{ remove_tags(item.name) }} + +