From fd4e755938ba65a6628581333f1be862752a18cd Mon Sep 17 00:00:00 2001 From: arabcoders Date: Sat, 24 Jan 2026 22:16:24 +0300 Subject: [PATCH] refactor: migrate tasks to db model --- app/features/core/schemas.py | 1 + .../definitions/handlers/_base_handler.py | 8 +- .../tasks/definitions/handlers/generic.py | 6 +- .../tasks/definitions/handlers/rss.py | 8 +- .../tasks/definitions/handlers/tver.py | 8 +- .../tasks/definitions/handlers/twitch.py | 8 +- .../tasks/definitions/handlers/youtube.py | 8 +- app/features/tasks/definitions/results.py | 226 ++++ app/features/tasks/definitions/service.py | 456 +++++++ .../tests/test_generic_task_handler.py | 7 +- .../definitions/tests/test_rss_handler.py | 47 +- .../definitions/tests/test_tver_handler.py | 11 +- .../definitions}/tests/test_twitch_handler.py | 9 +- .../tests/test_youtube_handler.py | 9 +- app/features/tasks/definitions/utils.py | 21 + app/features/tasks/deps.py | 14 + app/features/tasks/migration.py | 116 ++ app/features/tasks/models.py | 31 + app/features/tasks/repository.py | 169 +++ app/features/tasks/router.py | 480 +++++++ app/features/tasks/schemas.py | 131 ++ app/features/tasks/service.py | 185 +++ app/features/tasks/tests/__init__.py | 1 + .../tasks/tests/test_tasks_repository.py | 289 ++++ app/features/tasks/utils.py | 16 + app/library/Scheduler.py | 4 +- app/library/Services.py | 6 +- app/library/TaskDefinitions.py | 435 ------ app/library/Tasks.py | 1072 --------------- app/main.py | 2 +- .../20260124171740_add_tasks_table.py | 46 + app/routes/api/tasks.py | 460 +------ app/tests/test_tasks.py | 1183 ----------------- ui/app/components/TaskForm.vue | 12 +- ui/app/composables/useTasks.ts | 512 +++++++ ui/app/pages/tasks.vue | 342 ++--- ui/app/types/tasks.ts | 128 +- ui/app/utils/index.ts | 35 +- ui/tests/composables/useTasks.test.ts | 860 ++++++++++++ 39 files changed, 3910 insertions(+), 3452 deletions(-) create mode 100644 app/features/tasks/definitions/results.py create mode 100644 app/features/tasks/definitions/service.py rename app/{ => features/tasks/definitions}/tests/test_twitch_handler.py (81%) rename app/{ => features/tasks/definitions}/tests/test_youtube_handler.py (84%) create mode 100644 app/features/tasks/deps.py create mode 100644 app/features/tasks/migration.py create mode 100644 app/features/tasks/models.py create mode 100644 app/features/tasks/repository.py create mode 100644 app/features/tasks/router.py create mode 100644 app/features/tasks/schemas.py create mode 100644 app/features/tasks/service.py create mode 100644 app/features/tasks/tests/__init__.py create mode 100644 app/features/tasks/tests/test_tasks_repository.py create mode 100644 app/features/tasks/utils.py delete mode 100644 app/library/TaskDefinitions.py delete mode 100644 app/library/Tasks.py create mode 100644 app/migrations/20260124171740_add_tasks_table.py delete mode 100644 app/tests/test_tasks.py create mode 100644 ui/app/composables/useTasks.ts create mode 100644 ui/tests/composables/useTasks.test.ts diff --git a/app/features/core/schemas.py b/app/features/core/schemas.py index 03ff776b..6da489f5 100644 --- a/app/features/core/schemas.py +++ b/app/features/core/schemas.py @@ -20,6 +20,7 @@ class CEFeature(str, Enum): DL_FIELDS = "dl_fields" CONDITIONS = "conditions" NOTIFICATIONS = "notifications" + TASKS = "tasks" TASKS_DEFINITIONS = "tasks_definitions" def __str__(self) -> str: diff --git a/app/features/tasks/definitions/handlers/_base_handler.py b/app/features/tasks/definitions/handlers/_base_handler.py index dbe0880e..0ee1e274 100644 --- a/app/features/tasks/definitions/handlers/_base_handler.py +++ b/app/features/tasks/definitions/handlers/_base_handler.py @@ -4,22 +4,22 @@ from typing import Any import httpx from yt_dlp.utils.networking import random_user_agent +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult from app.library.config import Config from app.library.httpx_client import async_client -from app.library.Tasks import Task, TaskFailure, TaskResult class BaseHandler: @staticmethod - async def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: return False @staticmethod - async def extract(task: Task, config: Config | None = None) -> TaskResult | TaskFailure: + async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: raise NotImplementedError @classmethod - async def inspect(cls, task: Task, config: Config | None = None) -> TaskResult | TaskFailure: + async def inspect(cls, task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: return await cls.extract(task=task, config=config) @staticmethod diff --git a/app/features/tasks/definitions/handlers/generic.py b/app/features/tasks/definitions/handlers/generic.py index 5888e920..90e68a72 100644 --- a/app/features/tasks/definitions/handlers/generic.py +++ b/app/features/tasks/definitions/handlers/generic.py @@ -16,6 +16,7 @@ import jmespath from parsel import Selector from yt_dlp.utils.networking import random_user_agent +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.tasks.definitions.schemas import ( ExtractionRule, TaskDefinition, @@ -23,7 +24,6 @@ from app.features.tasks.definitions.schemas import ( from app.library.cache import Cache from app.library.config import Config from app.library.httpx_client import async_client -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult from app.library.Utils import fetch_info, get_archive_id from ._base_handler import BaseHandler @@ -112,7 +112,7 @@ class GenericTaskHandler(BaseHandler): return None @staticmethod - async def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: """ Determine if this handler can process the given task. @@ -131,7 +131,7 @@ class GenericTaskHandler(BaseHandler): return False @staticmethod - async def extract(task: Task, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004 + async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004 definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url) if not definition: return TaskFailure(message="No generic task definition matched the provided URL.") diff --git a/app/features/tasks/definitions/handlers/rss.py b/app/features/tasks/definitions/handlers/rss.py index f42dc99c..0f538a04 100644 --- a/app/features/tasks/definitions/handlers/rss.py +++ b/app/features/tasks/definitions/handlers/rss.py @@ -4,8 +4,8 @@ import re from typing import TYPE_CHECKING, Any from xml.etree.ElementTree import Element +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.library.cache import Cache -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult from app.library.Utils import fetch_info, get_archive_id from ._base_handler import BaseHandler @@ -24,13 +24,13 @@ class RssGenericHandler(BaseHandler): ) @staticmethod - async def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: LOG.debug(f"'{task.name}': Checking if task URL is parsable RSS feed: {task.url}") return RssGenericHandler.parse(task.url) is not None @staticmethod async def _get( - task: Task, + task: HandleTask, params: dict, parsed: dict[str, str], ) -> tuple[str, list[dict[str, str]], int]: @@ -131,7 +131,7 @@ class RssGenericHandler(BaseHandler): return feed_url, items, real_count @staticmethod - async def extract(task: Task) -> TaskResult | TaskFailure: + async def extract(task: HandleTask) -> TaskResult | TaskFailure: """ Extract items from an RSS/Atom feed. diff --git a/app/features/tasks/definitions/handlers/tver.py b/app/features/tasks/definitions/handlers/tver.py index f90797c6..81c2a4e8 100644 --- a/app/features/tasks/definitions/handlers/tver.py +++ b/app/features/tasks/definitions/handlers/tver.py @@ -1,7 +1,7 @@ import logging import re -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.library.Utils import get_archive_id from ._base_handler import BaseHandler @@ -21,7 +21,7 @@ class TverHandler(BaseHandler): RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?tver\.jp\/series\/(?Psr[a-z0-9_]+)$") @staticmethod - async def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: LOG.debug(f"Checking if task '{task.name}' is using parsable Tver series URL: {task.url}") return TverHandler.parse(task.url) is not None @@ -58,7 +58,7 @@ class TverHandler(BaseHandler): @staticmethod async def _collect_feed( - task: Task, + task: HandleTask, params: dict, series_id: str, ) -> tuple[str, list[dict[str, str]], bool]: @@ -142,7 +142,7 @@ class TverHandler(BaseHandler): return feed_url, items, has_items @staticmethod - async def extract(task: Task) -> TaskResult | TaskFailure: + async def extract(task: HandleTask) -> TaskResult | TaskFailure: series_id: str | None = TverHandler.parse(task.url) if not series_id: return TaskFailure(message="Unrecognized Tver series URL.") diff --git a/app/features/tasks/definitions/handlers/twitch.py b/app/features/tasks/definitions/handlers/twitch.py index 29baec19..25714bc2 100644 --- a/app/features/tasks/definitions/handlers/twitch.py +++ b/app/features/tasks/definitions/handlers/twitch.py @@ -3,7 +3,7 @@ import re from typing import TYPE_CHECKING from xml.etree.ElementTree import Element -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.library.Utils import get_archive_id from ._base_handler import BaseHandler @@ -20,13 +20,13 @@ class TwitchHandler(BaseHandler): RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?twitch\.tv\/(?P[a-z0-9_]{3,25})(?:\/.*)?$") @staticmethod - async def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: LOG.debug(f"Checking if task '{task.name}' is using parsable Twitch URL: {task.url}") return TwitchHandler.parse(task.url) is not None @staticmethod async def _collect_feed( - task: Task, + task: HandleTask, params: dict, handle_name: str, ) -> tuple[str, list[dict[str, str]], bool]: @@ -74,7 +74,7 @@ class TwitchHandler(BaseHandler): return feed_url, items, has_items @staticmethod - async def extract(task: Task) -> TaskResult | TaskFailure: + async def extract(task: HandleTask) -> TaskResult | TaskFailure: handle_name: str | None = TwitchHandler.parse(task.url) if not handle_name: return TaskFailure(message="Unrecognized Twitch channel URL.") diff --git a/app/features/tasks/definitions/handlers/youtube.py b/app/features/tasks/definitions/handlers/youtube.py index f29276d1..887ba78a 100644 --- a/app/features/tasks/definitions/handlers/youtube.py +++ b/app/features/tasks/definitions/handlers/youtube.py @@ -3,7 +3,7 @@ import re from typing import TYPE_CHECKING, Any from xml.etree.ElementTree import Element -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.library.Utils import get_archive_id from ._base_handler import BaseHandler @@ -26,12 +26,12 @@ class YoutubeHandler(BaseHandler): ) @staticmethod - async def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}") return YoutubeHandler.parse(task.url) is not None @staticmethod - async def _get(task: Task, params: dict, parsed: dict[str, str]) -> tuple[str, list[dict[str, str]], int]: + async def _get(task: HandleTask, params: dict, parsed: dict[str, str]) -> tuple[str, list[dict[str, str]], int]: """ Fetch the feed and return raw entries. @@ -89,7 +89,7 @@ class YoutubeHandler(BaseHandler): return feed_url, items, real_count @staticmethod - async def extract(task: Task) -> TaskResult | TaskFailure: + async def extract(task: HandleTask) -> TaskResult | TaskFailure: parsed: dict[str, str] | None = YoutubeHandler.parse(task.url) if not parsed: return TaskFailure(message="Unrecognized YouTube channel or playlist URL.") diff --git a/app/features/tasks/definitions/results.py b/app/features/tasks/definitions/results.py new file mode 100644 index 00000000..e9d293a9 --- /dev/null +++ b/app/features/tasks/definitions/results.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from app.features.tasks.schemas import Task as TaskSchema +from app.library.YTDLPOpts import YTDLPOpts + +from .utils import split_inspect_metadata + + +class HandleTask(TaskSchema): + def get_ytdlp_opts(self) -> YTDLPOpts: + """ + Get the yt-dlp options for the task. + + Returns: + YTDLPOpts: The yt-dlp options. + + """ + from app.library.YTDLPOpts import YTDLPOpts + + params: YTDLPOpts = YTDLPOpts.get_instance() + + if self.preset: + params = params.preset(name=self.preset) + + if self.cli: + params = params.add_cli(self.cli, from_user=True) + + if self.template: + params = params.add({"outtmpl": {"default": self.template}}, from_user=False) + + return params + + async def mark(self) -> tuple[bool, str]: + """ + Mark the task's items as downloaded in the archive file. + + Returns: + tuple[bool, str]: A tuple indicating success and a message. + + """ + from app.library.Utils import archive_add + + ret: tuple[bool, str] | dict[str, Any] = await self._mark_logic() + if isinstance(ret, tuple): + return ret + + archive_file: Path = ret.get("file") + items: set[str] = ret.get("items", set()) + + if len(items) < 1 or not archive_add(archive_file, list(items)): + return (True, "No new items to mark as downloaded.") + + return (True, f"Task '{self.name}' items marked as downloaded.") + + async def unmark(self) -> tuple[bool, str]: + """ + Unmark the task's items from the archive file. + + Returns: + tuple[bool, str]: A tuple indicating success and a message. + + """ + from app.library.Utils import archive_delete + + ret: tuple[bool, str] | dict[str, Any] = await self._mark_logic() + if isinstance(ret, tuple): + return ret + + archive_file: Path = ret.get("file") + items: set[str] = ret.get("items", set()) + + if len(items) < 1 or not archive_delete(archive_file, list(items)): + return (True, "No items to remove from archive file.") + + return (True, f"Removed '{self.name}' items from archive file.") + + async def fetch_metadata(self, full: bool = False) -> tuple[dict[str, Any] | None, bool, str]: + """ + Fetch metadata for the task's URL. + + Args: + full (bool): Whether to fetch full metadata including all entries for playlists. + + Returns: + tuple[dict[str, Any]|None, bool, str]: A tuple containing the metadata (or None on failure), a boolean + indicating if the operation was successful, and a message. + + """ + from app.library.ytdlp import fetch_info + + if not self.url: + return ({}, False, "No URL found in task parameters.") + + params = self.get_ytdlp_opts() + if not full: + params.add_cli("-I0", from_user=False) + + params_dict = params.get_all() + + ie_info: dict | None = await fetch_info( + params_dict, + self.url, + no_archive=True, + follow_redirect=False, + sanitize_info=True, + ) + + if not ie_info or not isinstance(ie_info, dict): + return ({}, False, "Failed to extract information from URL.") + + return (ie_info, True, "") + + async def _mark_logic(self) -> tuple[bool, str] | dict[str, Any]: + """ + Internal logic for marking/un-marking items. + + Returns: + tuple[bool, str] | dict[str, Any]: Either an error tuple or a dict with 'file' and 'items' keys. + + """ + from app.library.ytdlp import fetch_info + + if not self.url: + return (False, "No URL found in task parameters.") + + params: dict = self.get_ytdlp_opts().get_all() + if not (archive_file := params.get("download_archive")): + return (False, "No archive file found.") + + archive_file: Path = Path(archive_file) + + ie_info: dict | None = await fetch_info(params, self.url, no_archive=True, follow_redirect=True) + if not ie_info or not isinstance(ie_info, dict): + return (False, "Failed to extract information from URL.") + + if "playlist" != ie_info.get("_type"): + return (False, "Expected a playlist type from extract_info.") + + items: set[str] = set() + + def _process(item: dict): + for entry in item.get("entries", []): + if not isinstance(entry, dict): + continue + + if "playlist" == entry.get("_type"): + _process(entry) + continue + + if entry.get("_type") not in ("video", "url"): + continue + + if not entry.get("id") or not entry.get("ie_key"): + continue + + archive_id: str = f"{entry.get('ie_key', '').lower()} {entry.get('id')}" + + items.add(archive_id) + + _process(ie_info) + + return {"file": archive_file, "items": items} + + +@dataclass(slots=True) +class TaskItem: + """Represents a single item in a task result.""" + + url: str + "The URL of the item." + title: str | None = None + "The title of the item." + archive_id: str | None = None + "The archive ID of the item." + metadata: dict[str, Any] = field(default_factory=dict) + "Additional metadata related to the item." + + +@dataclass(slots=True) +class TaskResult: + """Represents a successful task handler execution result.""" + + items: list[TaskItem] = field(default_factory=list) + "The list of items." + metadata: dict[str, Any] = field(default_factory=dict) + "Additional metadata related to the result." + + def serialize(self) -> dict[str, Any]: + primary, extra = split_inspect_metadata(self.metadata) + payload: dict[str, Any] = {**primary, "items": [asdict(item) for item in self.items]} + + if extra: + payload["metadata"] = extra + + return payload + + +@dataclass(slots=True) +class TaskFailure: + """Represents a failed task handler execution result.""" + + message: str + "A human-readable message describing the failure." + error: str | None = None + "An optional error code or string." + metadata: dict[str, Any] = field(default_factory=dict) + "Additional metadata related to the failure." + + def serialize(self) -> dict[str, Any]: + primary, extra = split_inspect_metadata(self.metadata) + payload: dict[str, Any] = dict(primary) + + if self.error: + payload["error"] = self.error + + if self.message and (not self.error or self.message != self.error): + payload["message"] = self.message + + if extra: + payload["metadata"] = extra + + return payload diff --git a/app/features/tasks/definitions/service.py b/app/features/tasks/definitions/service.py new file mode 100644 index 00000000..f7702feb --- /dev/null +++ b/app/features/tasks/definitions/service.py @@ -0,0 +1,456 @@ +from __future__ import annotations + +import asyncio +import importlib +import inspect +import logging +import pkgutil +import random +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult +from app.features.tasks.models import TaskModel +from app.library.downloads.queue_manager import DownloadQueue +from app.library.Events import EventBus, Events +from app.library.ItemDTO import Item, ItemDTO +from app.library.Services import Services +from app.library.Utils import archive_read + +if TYPE_CHECKING: + from app.features.tasks.repository import TasksRepository + from app.library.config import Config + from app.library.Scheduler import Scheduler + +LOG: logging.Logger = logging.getLogger("tasks.definitions.service") + + +class TaskHandle: + def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None: + self._handlers: list[type] = [] + "The available handlers." + self._repo: TasksRepository = tasks + "The tasks manager." + self._scheduler: Scheduler = scheduler + "The scheduler." + self._config: Config = config + "The configuration." + self._task_name: str = f"{__class__.__name__}._dispatcher" + "The task name for the scheduler." + self._queued: dict[str, set[str]] = {} + "Queued archive IDs per handler." + self._failure_count: dict[str, dict[str, int]] = {} + "Failure counts per handler and archive ID." + + EventBus.get_instance().subscribe( + Events.ITEM_ERROR, + self._handle_item_error, + f"{__class__.__name__}.item_error", + ) + + def load(self) -> None: + self._handlers: list[type] = self._discover() + + timer: str = self._config.tasks_handler_timer + try: + from cronsim import CronSim + + CronSim(timer, datetime.now(UTC)) + except Exception as e: + timer = "15 */1 * * *" + LOG.error(f"Invalid timer format. '{e!s}'. Defaulting to '{timer}'.") + + self._scheduler.add( + timer=timer, + func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"), + id=f"{__class__.__name__}._dispatcher", + ) + + async def _dispatcher(self): + s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []} + + handler_groups: dict[str, list[tuple[HandleTask, type]]] = {} + + tasks: list[TaskModel] = await self._repo.list() + + for task_model in tasks: + task: HandleTask = HandleTask.model_validate(task_model) + + if not task.enabled or not task.handler_enabled: + s["d"].append(task.name) + continue + + if not task.get_ytdlp_opts().get_all().get("download_archive"): + LOG.debug(f"Task '{task.name}' does not have an archive file configured.") + s["f"].append(task.name) + continue + + try: + handler: type | None = await self._find_handler(task) + if handler is None: + s["u"].append(task.name) + continue + + handler_name: str = handler.__name__ + if handler_name not in handler_groups: + handler_groups[handler_name] = [] + handler_groups[handler_name].append((task, handler)) + s["h"].append(task.name) + except Exception as e: + LOG.error(f"Failed to handle task '{task.name}'. '{e!s}'.") + s["f"].append(task.name) + + for tasks_with_handlers in handler_groups.values(): + for idx, (task, handler) in enumerate(tasks_with_handlers): + try: + t: asyncio.Task[TaskResult | TaskFailure | None] = asyncio.create_task( + coro=self._dispatch( + task, + handler, + delay=0.0 if 0 == idx else random.uniform(1.0, self._config.task_handler_random_delay), + ), + name=f"taskHandler-{task.id}", + ) + t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t)) + except Exception as e: + LOG.error(f"Failed to dispatch task '{task.name}'. '{e!s}'.") + + if len(tasks) > 0: + LOG.info( + f"Tasks handler summary: Handled: {len(s['h'])}, Unhandled: {len(s['u'])}, Disabled: {len(s['d'])}, Failed: {len(s['f'])}." + ) + + async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None: + """ + Dispatch a task after a random delay to avoid rate limiting. + + Args: + task: The task to dispatch. + handler: The handler to use. + delay: The delay in seconds before dispatching. + + Returns: + The dispatch result. + + """ + if delay > 0: + LOG.debug(f"Delaying dispatch of task '{task.name}' by {delay:.1f} seconds.") + await asyncio.sleep(delay) + return await self.dispatch(task, handler=handler) + + def _handle_exception(self, fut: asyncio.Task, task: HandleTask) -> None: + if fut.cancelled(): + return + + if exc := fut.exception(): + LOG.error(f"Exception while handling task '{task.name}': {exc}") + + async def _find_handler(self, task: HandleTask) -> type | None: + for cls in self._handlers: + try: + if await Services.get_instance().handle_async(handler=cls.can_handle, task=task): + return cls + except Exception as e: + LOG.exception(e) + continue + + return None + + async def dispatch( + self, + task: HandleTask, + handler: type | None = None, + **kwargs, # noqa: ARG002 + ) -> TaskResult | TaskFailure | None: + """ + Dispatch a task to the appropriate handler. + + Args: + task: The task to dispatch. + handler: Optional specific handler to use instead of finding one. + **kwargs: Additional context to pass to the handler. + + Returns: + The extraction outcome, or None if no handler matched. + + """ + if not handler: + handler = await self._find_handler(task) + if handler is None: + return None + + services: Services = Services.get_instance() + + try: + extraction: TaskResult | TaskFailure = await services.handle_async( + handler=handler.extract, task=task, config=self._config + ) + except NotImplementedError: + LOG.error(f"Handler '{handler.__name__}' does not implement extract().") + return TaskFailure(message="Handler does not support extraction.") + except Exception as exc: + LOG.exception(exc) + raise + + if isinstance(extraction, TaskFailure): + LOG.error(f"Handler '{handler.__name__}' failed to extract items: {extraction.message}") + return extraction + + if not isinstance(extraction, TaskResult): + LOG.error( + f"Handler '{handler.__name__}' returned unexpected result type '{type(extraction).__name__}'.", + ) + return TaskFailure( + message="Handler returned invalid result type.", metadata={"type": type(extraction).__name__} + ) + + raw_items: list[TaskItem] = extraction.items or [] + metadata: dict[str, Any] = extraction.metadata or {} + + handler_name: str = handler.__name__ + queued: set[str] = self._queued.setdefault(handler_name, set()) + failures: dict[str, int] = self._failure_count.setdefault(handler_name, {}) + + params: dict = task.get_ytdlp_opts().get_all() + archive_file: str | None = params.get("download_archive") + + download_queue: DownloadQueue = services.get("queue") or DownloadQueue.get_instance() + notify: EventBus = services.get("notify") or EventBus.get_instance() + + archive_ids: list[str] = [ + item.archive_id for item in raw_items if isinstance(item, TaskItem) and item.archive_id + ] + downloaded: list[str] = archive_read(archive_file, archive_ids) if archive_file else [] + + filtered: list[TaskItem] = [] + + for item in raw_items: + if not isinstance(item, TaskItem): + LOG.warning("Handler '{handler.__name__}' produced non-TaskItem entry: {item!r}") + continue + + url: str = item.url + if not url: + continue + + archive_id: str | None = item.archive_id + if not archive_id: + LOG.warning(f"'{task.name}': Item with URL '{url}' is missing an archive ID. Skipping.") + continue + + if archive_id in queued: + continue + + queued.add(archive_id) + + if archive_file and archive_id in downloaded: + continue + + if await download_queue.queue.exists(url=url): + continue + + try: + done = await download_queue.done.get(url=url) + if "error" != done.info.status: + continue + except KeyError: + pass + + if archive_id not in failures: + failures[archive_id] = 0 + + filtered.append(item) + + if not filtered: + if raw_items: + LOG.debug( + f"Handler '{handler.__name__}' produced '{len(raw_items)}' for '{task.name}' items, none queued after filtering." + ) + return TaskResult(items=[], metadata=metadata) + + LOG.info( + f"Handler '{handler.__name__}' Found '{len(filtered)}' new items for '{task.name}' (raw={len(raw_items)})." + ) + + base_item = Item.format( + { + "url": task.url, + "preset": task.preset or self._config.default_preset, + "folder": task.folder or "", + "template": task.template or "", + "cli": task.cli or "", + "auto_start": task.auto_start, + "extras": {"source_name": task.name, "source_id": task.id, "source_handler": handler.__name__}, + } + ) + + for item in filtered: + metadata_entry: dict[str, Any] = item.metadata if isinstance(item.metadata, dict) else {} + extras: dict[str, Any] = base_item.extras.copy() + if metadata_entry: + extras["metadata"] = metadata_entry + + notify.emit( + Events.ADD_URL, + data=base_item.new_with(url=item.url, extras=extras).serialize(), + ) + + return TaskResult(items=filtered, metadata=metadata) + + async def inspect( + self, + url: str, + preset: str | None = None, + handler_name: str | None = None, + static_only: bool = False, + ) -> TaskResult | TaskFailure: + """ + Inspect a URL to find a matching handler and optionally extract items. + + Args: + url: The URL to inspect. + preset: Optional preset name to use. + handler_name: Optional specific handler name to use. + static_only: If True, only check if a handler matches without extraction. + + Returns: + TaskResult or TaskFailure with inspection results. + + """ + if not self._handlers: + self._handlers = self._discover() + + task = HandleTask( + id=None, + name="Inspector", + url=url, + preset=preset or self._config.default_preset, + auto_start=False, + ) + + services = Services.get_instance() + + handler_cls: type | None + if handler_name: + handler_cls = next((cls for cls in self._handlers if cls.__name__.lower() == handler_name.lower()), None) + if handler_cls is None: + message: str = f"Handler '{handler_name}' not found." + return TaskFailure( + message=message, + error=message, + metadata={"matched": False, "handler": handler_name}, + ) + + try: + matched = await services.handle_async(handler=handler_cls.can_handle, task=task) + except Exception as exc: # pragma: no cover - defensive + LOG.exception(exc) + message = str(exc) + return TaskFailure( + message=message, + error=message, + metadata={"matched": False, "handler": handler_cls.__name__}, + ) + + if not matched: + return TaskFailure( + message="Handler cannot process the supplied URL.", + metadata={"matched": False, "handler": handler_cls.__name__}, + ) + else: + handler_cls = await self._find_handler(task) + if handler_cls is None: + message = "No handler matched the supplied URL." + return TaskFailure( + message=message, + error=message, + metadata={"matched": False, "handler": None}, + ) + + base_metadata: dict[str, Any] = {"matched": True, "handler": handler_cls.__name__} + + if static_only: + return TaskResult(items=[], metadata=base_metadata) + + try: + extraction: TaskResult | TaskFailure = await services.handle_async( + handler=handler_cls.extract, task=task, config=self._config + ) + except NotImplementedError: + return TaskFailure( + message="Handler does not support manual inspection.", + metadata={**base_metadata, "supported": False}, + ) + except Exception as exc: + LOG.exception(exc) + message = str(exc) + return TaskFailure( + message=message, + error=message, + metadata={**base_metadata, "supported": True}, + ) + + if isinstance(extraction, TaskFailure): + combined_failure_metadata: dict[str, Any] = {**base_metadata, "supported": True} + if extraction.metadata: + combined_failure_metadata.update(extraction.metadata) + + return TaskFailure( + message=extraction.message, + error=extraction.error if extraction.error else extraction.message, + metadata=combined_failure_metadata, + ) + + if not isinstance(extraction, TaskResult): + LOG.error( + f"Handler '{handler_cls.__name__}' returned unexpected result type '{type(extraction).__name__}' during inspection.", + ) + extraction = TaskResult() + + combined_metadata: dict[str, Any] = {**base_metadata, "supported": True} + if extraction.metadata: + combined_metadata.update(extraction.metadata) + + return TaskResult(items=list(extraction.items), metadata=combined_metadata) + + def _discover(self) -> list[type]: + """Discover all available task handlers.""" + import app.features.tasks.definitions.handlers as handlers_pkg + + handlers: list[type] = [] + + for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__): + if module_name.startswith("_"): + continue + + module = importlib.import_module(f"{handlers_pkg.__name__}.{module_name}") + for _, cls in inspect.getmembers(module, inspect.isclass): + if cls.__module__ != module.__name__: + continue + + if callable(getattr(cls, "can_handle", None)) and callable(getattr(cls, "extract", None)): + handlers.append(cls) + + return handlers + + async def _handle_item_error(self, event, _name, **_kwargs): + """Handle item error events to clean up queued items and track failures.""" + item: ItemDTO | None = getattr(event, "data", None) + if not isinstance(item, ItemDTO): + return + + extras: dict[Any, Any] = getattr(item, "extras", {}) or {} + handler_name: Any | None = extras.get("source_handler") + if not handler_name: + return + + archive_id: str | None = item.archive_id + if not archive_id: + return + + queued: set[str] | None = self._queued.get(handler_name) + if queued: + queued.discard(archive_id) + + failures: dict[str, int] = self._failure_count.setdefault(handler_name, {}) + failures[archive_id] = failures.get(archive_id, 0) + 1 diff --git a/app/features/tasks/definitions/tests/test_generic_task_handler.py b/app/features/tasks/definitions/tests/test_generic_task_handler.py index 31fd4161..b55f78e2 100644 --- a/app/features/tasks/definitions/tests/test_generic_task_handler.py +++ b/app/features/tasks/definitions/tests/test_generic_task_handler.py @@ -4,14 +4,15 @@ from unittest.mock import patch import pytest from app.features.tasks.definitions.handlers.generic import GenericTaskHandler +from app.features.tasks.definitions.results import TaskFailure, TaskResult from app.features.tasks.definitions.schemas import ( + Definition, EngineConfig, RequestConfig, ResponseConfig, TaskDefinition, - Definition, ) -from app.library.Tasks import Task, TaskFailure, TaskResult +from app.features.tasks.definitions.results import HandleTask @pytest.fixture(autouse=True) @@ -341,7 +342,7 @@ async def test_generic_task_handler_inspect(monkeypatch): return {"id": "test_video_1", "extractor_key": "Example"} with patch("app.features.tasks.definitions.handlers.generic.fetch_info", side_effect=fake_fetch_info): - task = Task(id="inspect", name="Inspect", url="https://example.com/api") + task = HandleTask(id=1, name="Inspect", url="https://example.com/api") result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task) assert isinstance(result, TaskResult), "Result should be TaskResult" diff --git a/app/features/tasks/definitions/tests/test_rss_handler.py b/app/features/tasks/definitions/tests/test_rss_handler.py index be241892..04333094 100644 --- a/app/features/tasks/definitions/tests/test_rss_handler.py +++ b/app/features/tasks/definitions/tests/test_rss_handler.py @@ -1,7 +1,8 @@ import pytest from app.features.tasks.definitions.handlers.rss import RssGenericHandler -from app.library.Tasks import Task, TaskResult +from app.features.tasks.definitions.results import TaskResult +from app.features.tasks.definitions.results import HandleTask class DummyResponse: @@ -68,10 +69,10 @@ class TestRssHandlerExtraction: return DummyResponse(atom_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_rss", + task = HandleTask( + id=1, name="Test Atom Feed", url="https://example.com/feed.atom", preset="default", @@ -112,10 +113,10 @@ class TestRssHandlerExtraction: return DummyResponse(rss_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_rss", + task = HandleTask( + id=1, name="Test RSS Feed", url="https://example.com/feed.rss", preset="default", @@ -146,19 +147,19 @@ class TestRssHandlerExtraction: return DummyResponse(atom_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_rss", - name="Test Feed", + task = HandleTask( + id=1, + name="Test rss Feed", url="https://example.com/feed.atom", preset="default", ) assert await RssGenericHandler.can_handle(task) is True - non_feed_task = Task( - id="test_youtube", + non_feed_task = HandleTask( + id=1, name="YouTube Video", url="https://www.youtube.com/watch?v=abc123", preset="default", @@ -185,10 +186,10 @@ class TestRssHandlerEdgeCases: return DummyResponse(empty_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_empty", + task = HandleTask( + id=1, name="Empty Feed", url="https://example.com/feed.rss", preset="default", @@ -203,17 +204,17 @@ class TestRssHandlerEdgeCases: @pytest.mark.asyncio async def test_invalid_feed_url(self, monkeypatch): """Test handling of invalid feed URL.""" - from app.library.Tasks import TaskFailure + from app.features.tasks.definitions.results import TaskFailure async def fake_request(**kwargs): # noqa: ARG001 msg = "Network error" raise Exception(msg) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_invalid", + task = HandleTask( + id=1, name="Invalid Feed", url="https://example.com/feed.rss", preset="default", @@ -244,10 +245,10 @@ class TestRssHandlerEdgeCases: return DummyResponse(feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_missing", + task = HandleTask( + id=1, name="Feed with Missing URLs", url="https://example.com/feed.rss", preset="default", diff --git a/app/features/tasks/definitions/tests/test_tver_handler.py b/app/features/tasks/definitions/tests/test_tver_handler.py index 2b8c6442..b2179c1a 100644 --- a/app/features/tasks/definitions/tests/test_tver_handler.py +++ b/app/features/tasks/definitions/tests/test_tver_handler.py @@ -1,7 +1,8 @@ import pytest from app.features.tasks.definitions.handlers.tver import TverHandler -from app.library.Tasks import Task, TaskResult +from app.features.tasks.definitions.results import TaskResult +from app.features.tasks.definitions.results import HandleTask class DummyResponse: @@ -118,9 +119,9 @@ async def test_tver_handler_extract(monkeypatch): raise RuntimeError(msg) monkeypatch.setattr(TverHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"})) + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"})) - task = Task(id="test_tver", name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default") + task = HandleTask(id=1, name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default") result = await TverHandler.extract(task) @@ -154,7 +155,7 @@ def test_tver_handler_parse(url: str, should_match: bool): @pytest.mark.asyncio async def test_tver_handler_can_handle(): """Test tver handler can_handle method.""" - task_valid = Task(id="test1", name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default") - task_invalid = Task(id="test2", name="Test", url="https://youtube.com/watch?v=123", preset="default") + task_valid = HandleTask(id=1, name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default") + task_invalid = HandleTask(id=2, name="Test", url="https://youtube.com/watch?v=123", preset="default") assert await TverHandler.can_handle(task_valid) is True assert await TverHandler.can_handle(task_invalid) is False diff --git a/app/tests/test_twitch_handler.py b/app/features/tasks/definitions/tests/test_twitch_handler.py similarity index 81% rename from app/tests/test_twitch_handler.py rename to app/features/tasks/definitions/tests/test_twitch_handler.py index e5b0fc86..642c751c 100644 --- a/app/tests/test_twitch_handler.py +++ b/app/features/tasks/definitions/tests/test_twitch_handler.py @@ -1,7 +1,8 @@ import pytest from app.features.tasks.definitions.handlers.twitch import TwitchHandler -from app.library.Tasks import Task, TaskResult +from app.features.tasks.definitions.results import TaskResult +from app.features.tasks.definitions.results import HandleTask class DummyResponse: @@ -39,10 +40,10 @@ async def test_twitch_handler_inspect(monkeypatch): return DummyResponse(feed) monkeypatch.setattr(TwitchHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="inspect", + task = HandleTask( + id=1, name="Inspect", url="https://www.twitch.tv/testchannel", preset="default", diff --git a/app/tests/test_youtube_handler.py b/app/features/tasks/definitions/tests/test_youtube_handler.py similarity index 84% rename from app/tests/test_youtube_handler.py rename to app/features/tasks/definitions/tests/test_youtube_handler.py index 26c49db1..3482eefa 100644 --- a/app/tests/test_youtube_handler.py +++ b/app/features/tasks/definitions/tests/test_youtube_handler.py @@ -1,7 +1,8 @@ import pytest from app.features.tasks.definitions.handlers.youtube import YoutubeHandler -from app.library.Tasks import Task, TaskResult +from app.features.tasks.definitions.results import TaskResult +from app.features.tasks.definitions.results import HandleTask class DummyResponse: @@ -41,10 +42,10 @@ async def test_youtube_handler_inspect(monkeypatch): return DummyResponse(feed) monkeypatch.setattr(YoutubeHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="inspect", + task = HandleTask( + id=1, name="Inspect", url="https://www.youtube.com/channel/UCabcdefghijklmnopqrstuv", preset="default", diff --git a/app/features/tasks/definitions/utils.py b/app/features/tasks/definitions/utils.py index 1a64dbda..c1fba3d8 100644 --- a/app/features/tasks/definitions/utils.py +++ b/app/features/tasks/definitions/utils.py @@ -46,3 +46,24 @@ def schema_to_payload(item: TaskDefinition) -> dict[str, Any]: "enabled": item.enabled, "definition": item.definition.model_dump(exclude_unset=True, exclude_none=True), } + + +def split_inspect_metadata(metadata: dict[str, Any] | None) -> tuple[dict[str, Any], dict[str, Any]]: + """ + Split commonly consumed metadata keys from the rest. + + Args: + metadata (dict[str, Any]|None): The metadata to split. + + Returns: + tuple[dict[str, Any], dict[str, Any]]: The primary and extra metadata. + + """ + metadata = dict(metadata or {}) + primary: dict[str, Any] = {} + + for key in ("matched", "handler", "supported"): + if key in metadata: + primary[key] = metadata.pop(key) + + return primary, metadata diff --git a/app/features/tasks/deps.py b/app/features/tasks/deps.py new file mode 100644 index 00000000..eef934ba --- /dev/null +++ b/app/features/tasks/deps.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from app.features.tasks.repository import TasksRepository +from app.features.tasks.service import Tasks + + +def get_tasks_repo() -> TasksRepository: + """Get tasks repository instance.""" + return TasksRepository.get_instance() + + +def get_tasks_service() -> Tasks: + """Get tasks service instance.""" + return Tasks.get_instance() diff --git a/app/features/tasks/migration.py b/app/features/tasks/migration.py new file mode 100644 index 00000000..77e4928a --- /dev/null +++ b/app/features/tasks/migration.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from app.features.core.migration import Migration as FeatureMigration +from app.features.tasks.schemas import Task +from app.library.config import Config + +if TYPE_CHECKING: + from app.features.tasks.repository import TasksRepository + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Migration(FeatureMigration): + name: str = "tasks" + + def __init__(self, repo: TasksRepository, config: Config | None = None): + self._config: Config = config or Config.get_instance() + super().__init__(config=self._config) + self._repo: TasksRepository = repo + self._source_file: Path = Path(self._config.config_path) / "tasks.json" + + async def should_run(self) -> bool: + return self._source_file.exists() + + async def migrate(self) -> None: + if await self._repo.count() > 0: + LOG.warning("Tasks already exist in the database; skipping migration.") + await self._move_file(self._source_file) + return + + try: + items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text()) + except Exception as exc: + LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc) + await self._move_file(self._source_file) + return + + if items is None: + LOG.warning("No tasks found in %s; skipping migration.", self._source_file) + await self._move_file(self._source_file) + return + + inserted = 0 + seen_names: dict[str, int] = {} + for index, item in enumerate(items): + if not (normalized := self._normalize(item, index, seen_names)): + continue + try: + await self._repo.create(normalized) + inserted += 1 + except Exception as exc: + LOG.exception("Failed to insert task '%s': %s", normalized["name"], exc) + + LOG.info("Migrated %s task(s) from %s.", inserted, self._source_file) + await self._move_file(self._source_file) + + def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> dict[str, Any] | None: + if not isinstance(item, dict): + LOG.warning("Skipping task at index %s due to invalid type.", index) + return None + + assert isinstance(item, dict) + + name: str | None = item.get("name") + if not name or not isinstance(name, str): + LOG.warning("Skipping task at index %s due to missing name.", index) + return None + + normalized_name = name.strip() + if not normalized_name: + LOG.warning("Skipping task at index %s due to empty name.", index) + return None + + name = self._unique_name(normalized_name, seen_names) + + url: str | None = item.get("url") + if not url or not isinstance(url, str): + LOG.warning("Skipping task '%s' at index %s due to missing URL.", name, index) + return None + + url = url.strip() + if not url: + LOG.warning("Skipping task '%s' at index %s due to empty URL.", name, index) + return None + + folder: str = item.get("folder") if isinstance(item.get("folder"), str) else "" + preset: str = item.get("preset") if isinstance(item.get("preset"), str) else "" + timer: str = item.get("timer") if isinstance(item.get("timer"), str) else "" + template: str = item.get("template") if isinstance(item.get("template"), str) else "" + cli: str = item.get("cli") if isinstance(item.get("cli"), str) else "" + auto_start: bool = item.get("auto_start") if isinstance(item.get("auto_start"), bool) else True + handler_enabled: bool = item.get("handler_enabled") if isinstance(item.get("handler_enabled"), bool) else True + enabled: bool = item.get("enabled") if isinstance(item.get("enabled"), bool) else True + + try: + validated = Task( + name=name, + url=url, + folder=folder, + preset=preset, + timer=timer, + template=template, + cli=cli, + auto_start=auto_start, + handler_enabled=handler_enabled, + enabled=enabled, + ) + return validated.model_dump() + except Exception as e: + LOG.warning("Skipping task '%s' at index %s due to validation error: %s", name, index, e) + return None diff --git a/app/features/tasks/models.py b/app/features/tasks/models.py new file mode 100644 index 00000000..9993025d --- /dev/null +++ b/app/features/tasks/models.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from datetime import datetime # noqa: TC003 + +from sqlalchemy import Boolean, Index, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.features.core.models import Base, UTCDateTime, utcnow + + +class TaskModel(Base): + __tablename__: str = "tasks" + __table_args__: tuple[Index, ...] = ( + Index("ix_tasks_name", "name"), + Index("ix_tasks_enabled", "enabled"), + Index("ix_tasks_timer", "timer"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + url: Mapped[str] = mapped_column(String(2048), nullable=False) + folder: Mapped[str] = mapped_column(String(512), nullable=False, default="") + preset: Mapped[str] = mapped_column(String(255), nullable=False, default="") + timer: Mapped[str] = mapped_column(String(255), nullable=False, default="") + template: Mapped[str] = mapped_column(String(1024), nullable=False, default="") + cli: Mapped[str] = mapped_column(Text, nullable=False, default="") + auto_start: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + handler_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False) + updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False) diff --git a/app/features/tasks/repository.py b/app/features/tasks/repository.py new file mode 100644 index 00000000..4af8ede5 --- /dev/null +++ b/app/features/tasks/repository.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, or_, select + +from app.features.core.deps import get_session +from app.features.tasks.models import TaskModel +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from sqlalchemy.engine.result import Result + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Select + +LOG: logging.Logger = logging.getLogger(__name__) + + +class TasksRepository(metaclass=Singleton): + def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + self._migrated = False + self.session = session or get_session + + async def run_migrations(self) -> None: + if self._migrated: + return + + self._migrated = True + from app.features.tasks.migration import Migration + + await Migration(repo=self).run() + + @staticmethod + def get_instance() -> TasksRepository: + return TasksRepository() + + async def list(self) -> list[TaskModel]: + async with self.session() as session: + result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc())) + return list(result.scalars().all()) + + async def list_paginated(self, page: int, per_page: int) -> tuple[list[TaskModel], int, int, int]: + async with self.session() as session: + total: int = await self.count() + total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1 + + if page > total_pages and total > 0: + page = total_pages + + query: Select[tuple[TaskModel]] = ( + select(TaskModel).order_by(TaskModel.name.asc()).limit(per_page).offset((page - 1) * per_page) + ) + result: Result[tuple[TaskModel]] = await session.execute(query) + return list(result.scalars().all()), total, page, total_pages + + async def count(self) -> int: + async with self.session() as session: + result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(TaskModel)) + return int(result.scalar_one()) + + async def get(self, identifier: int | str) -> TaskModel | None: + async with self.session() as session: + if not identifier: + return None + + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskModel.id == int(identifier), TaskModel.name == identifier) + else: + clause = TaskModel.name == identifier + + result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).where(clause).limit(1)) + return result.scalar_one_or_none() + + async def get_by_name(self, name: str, exclude_id: int | None = None) -> TaskModel | None: + async with self.session() as session: + query: Select[tuple[TaskModel]] = select(TaskModel).where(TaskModel.name == name) + if exclude_id is not None: + query = query.where(TaskModel.id != exclude_id) + + result: Result[tuple[TaskModel]] = await session.execute(query.limit(1)) + return result.scalar_one_or_none() + + async def get_all_enabled(self) -> list[TaskModel]: + """Get all enabled tasks.""" + async with self.session() as session: + result: Result[tuple[TaskModel]] = await session.execute( + select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc()) # noqa: E712 + ) + return list(result.scalars().all()) + + async def get_all_with_timer(self) -> list[TaskModel]: + """Get all tasks that have a timer configured.""" + async with self.session() as session: + result: Result[tuple[TaskModel]] = await session.execute( + select(TaskModel).where(TaskModel.timer != "").order_by(TaskModel.name.asc()) + ) + return list(result.scalars().all()) + + async def create(self, payload: TaskModel | dict) -> TaskModel: + async with self.session() as session: + model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload + if model.id is not None: + model.id = None + + if await self.get_by_name(name=model.name) is not None: + msg: str = f"Task with name '{model.name}' already exists." + raise ValueError(msg) + + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def update(self, identifier: int | str, payload: dict[str, Any]) -> TaskModel: + """Update an existing task.""" + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskModel.id == int(identifier), TaskModel.name == identifier) + else: + clause = TaskModel.name == identifier + + result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).where(clause).limit(1)) + model: TaskModel | None = result.scalar_one_or_none() + + if model is None: + msg: str = f"Task '{identifier}' not found." + raise KeyError(msg) + + if "name" in payload: + existing: TaskModel | None = await self.get_by_name(name=payload["name"], exclude_id=model.id) + if existing is not None: + msg = f"Task with name '{payload['name']}' already exists." + raise ValueError(msg) + + for key, value in payload.items(): + if hasattr(model, key): + setattr(model, key, value) + + await session.commit() + await session.refresh(model) + return model + + async def delete(self, identifier: int | str) -> TaskModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskModel.id == int(identifier), TaskModel.name == identifier) + else: + clause = TaskModel.name == identifier + + result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).where(clause).limit(1)) + model: TaskModel | None = result.scalar_one_or_none() + + if model is None: + msg: str = f"Task '{identifier}' not found." + raise KeyError(msg) + + await session.delete(model) + await session.commit() + return model diff --git a/app/features/tasks/router.py b/app/features/tasks/router.py new file mode 100644 index 00000000..6c086ee5 --- /dev/null +++ b/app/features/tasks/router.py @@ -0,0 +1,480 @@ +import logging +from typing import TYPE_CHECKING, Any + +from aiohttp import web +from aiohttp.web import Request, Response +from pydantic import ValidationError + +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination +from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination +from app.features.tasks.definitions.results import HandleTask as ExtendedTask +from app.features.tasks.definitions.results import TaskFailure, TaskResult +from app.features.tasks.definitions.service import TaskHandle +from app.features.tasks.repository import TasksRepository +from app.features.tasks.schemas import Task, TaskList, TaskPatch +from app.library.ag_utils import ag +from app.library.config import Config +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.router import route +from app.library.Utils import get_channel_images, get_file, parse_outtmpl, validate_url + +if TYPE_CHECKING: + from pathlib import Path + + +LOG: logging.Logger = logging.getLogger(__name__) + + +def _model(model: Any) -> Task: + return Task.model_validate(model) + + +def _serialize(model: Any) -> dict: + return _model(model).model_dump() + + +@route("GET", "api/tasks/", "tasks_list") +async def tasks_list(request: Request, repo: TasksRepository, encoder: Encoder) -> Response: + page, per_page = normalize_pagination(request) + items, total, current_page, total_pages = await repo.list_paginated(page, per_page) + return web.json_response( + data=TaskList( + items=[_model(model) for model in items], + pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)), + ), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/tasks/", "tasks_add") +async def tasks_add(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response: + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + item: Task = Task.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if await repo.get_by_name(item.name): + return web.json_response( + data={"error": f"Task with name '{item.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + try: + created = await repo.create(item.model_dump()) + saved = _serialize(created) + except ValueError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code) + + notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.CREATE, data=saved)) + + return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("GET", r"api/tasks/{id:\d+}", "tasks_get") +async def tasks_get(request: Request, repo: TasksRepository, encoder: Encoder) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + model = await repo.get(identifier) + if not model: + return web.json_response({"error": "Task not found"}, status=web.HTTPNotFound.status_code) + + return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("DELETE", r"api/tasks/{id:\d+}", "tasks_delete") +async def tasks_delete(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + try: + deleted = _serialize(await repo.delete(identifier)) + notify.emit( + Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.DELETE, data=deleted) + ) + return web.json_response( + data=deleted, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + except KeyError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) + + +@route("PATCH", r"api/tasks/{id:\d+}", "tasks_patch") +async def tasks_patch(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + model = await repo.get(identifier) + if not model: + return web.json_response({"error": "Task not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = TaskPatch.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"Task with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True))) + notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated)) + return web.json_response( + data=updated, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("PUT", r"api/tasks/{id:\d+}", "tasks_update") +async def tasks_update(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + model = await repo.get(identifier) + if not model: + return web.json_response({"error": "Task not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = Task.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"Task with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True))) + notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated)) + + return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/tasks/inspect", "task_handler_inspect") +async def task_handler_inspect(request: Request, handler: TaskHandle, encoder: Encoder, config: Config) -> Response: + """ + Check if handler can process the given URL. + + Args: + request: The request object. + handler: The handler service instance. + encoder: The encoder instance. + config: The config instance. + + Returns: + The response object. + + """ + data = await request.json() + + url: str | None = data.get("url") if isinstance(data, dict) else None + if not url: + return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code) + + static_only: bool = data.get("static_only", False) if isinstance(data, dict) else False + if not static_only: + try: + validate_url(url, allow_internal=config.allow_internal_urls) + except ValueError as e: + return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code) + + preset: str = data.get("preset", "") if isinstance(data, dict) else "" + handler_name: str | None = data.get("handler") if isinstance(data, dict) else None + + try: + result: TaskResult | TaskFailure = await handler.inspect( + url=url, preset=preset, handler_name=handler_name, static_only=static_only + ) + except Exception as e: + LOG.exception(e) + return web.json_response( + {"error": "Failed to inspect handler.", "message": str(e)}, + status=web.HTTPInternalServerError.status_code, + ) + + return web.json_response( + data=result, + status=web.HTTPBadRequest.status_code if isinstance(result, TaskFailure) else web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", r"api/tasks/{id:\d+}/mark", "tasks_mark") +async def task_mark(request: Request, repo: TasksRepository, encoder: Encoder) -> Response: + """ + Mark all items from task as downloaded. + + Args: + request: The request object. + repo: The tasks repository instance. + encoder: The encoder instance. + + Returns: + The response object. + + """ + if not (task_id := request.match_info.get("id")): + return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) + + try: + model = await repo.get(int(task_id)) + if not model: + return web.json_response( + data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code + ) + + # Convert to extended Task with handler methods + task = ExtendedTask.model_validate(model) + _status, _message = await task.mark() + + if not _status: + return web.json_response(data={"error": _message}, status=web.HTTPBadRequest.status_code) + + return web.json_response(data={"message": _message}, status=web.HTTPOk.status_code, dumps=encoder.encode) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + + +@route("DELETE", r"api/tasks/{id:\d+}/mark", "tasks_unmark") +async def task_unmark(request: Request, repo: TasksRepository, encoder: Encoder) -> Response: + """ + Remove all task items from download archive. + + Args: + request: The request object. + repo: The tasks repository instance. + encoder: The encoder instance. + + Returns: + The response object. + + """ + if not (task_id := request.match_info.get("id")): + return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) + + try: + model = await repo.get(int(task_id)) + if not model: + return web.json_response( + data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code + ) + + # Convert to extended Task with handler methods + task = ExtendedTask.model_validate(model) + _status, _message = await task.unmark() + + if not _status: + return web.json_response(data={"error": _message}, status=web.HTTPBadRequest.status_code) + + return web.json_response(data={"message": _message}, status=web.HTTPOk.status_code, dumps=encoder.encode) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + + +@route("POST", r"api/tasks/{id:\d+}/metadata", "tasks_metadata") +async def task_metadata(request: Request, repo: TasksRepository, config: Config, encoder: Encoder) -> Response: + """ + Generate metadata for the task. + + Args: + request: The request object. + repo: The tasks repository instance. + config: The config instance. + encoder: The encoder instance. + + Returns: + The response object. + + """ + task_id = request.match_info.get("id") + + try: + if not (model := await repo.get(int(task_id))): + return web.json_response( + data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code + ) + + # Convert to extended Task with handler methods + task = ExtendedTask.model_validate(model) + + (save_path, _) = get_file(config.download_path, task.folder) + if not str(save_path or "").startswith(str(config.download_path)): + return web.json_response(data={"error": "Invalid task folder."}, status=web.HTTPBadRequest.status_code) + + if not save_path.exists(): + save_path.mkdir(parents=True, exist_ok=True) + + metadata, status, message = await task.fetch_metadata() + if not status: + return web.json_response(data={"error": message}, status=web.HTTPBadRequest.status_code) + + if not task.folder: + try: + ytdlp_opts: dict = task.get_ytdlp_opts().get_all() + outtmpl: str = parse_outtmpl( + output_template=ytdlp_opts.get("outtmpl", {}).get("default", "{title} [{id}]"), + info_dict=metadata, + params=ytdlp_opts, + ) + if outtmpl: + _path: Path = save_path / outtmpl + if not _path.is_dir(): + _path: Path = _path.parent + + (save_path, _) = get_file(config.download_path, _path.relative_to(config.download_path)) + if not str(save_path or "").startswith(str(config.download_path)): + return web.json_response( + data={"error": "Invalid final path folder."}, status=web.HTTPBadRequest.status_code + ) + + if not save_path.exists(): + save_path.mkdir(parents=True, exist_ok=True) + except Exception as e: + LOG.warning(f"Failed to resolve final path from outtmpl. '{e!s}'") + + info = { + "id": ag(metadata, ["id", "channel_id"]), + "id_type": metadata.get("extractor", "").split(":")[0].lower() if metadata.get("extractor") else None, + "title": ag(metadata, ["title", "fulltitle"]) or None, + "description": metadata.get("description", ""), + "uploader": metadata.get("uploader", ""), + "tags": metadata.get("tags", []), + "year": metadata.get("release_year"), + "thumbnails": get_channel_images(metadata.get("thumbnails", {})), + } + + if not info.get("title"): + return web.json_response( + data={"error": "Failed to get title from metadata."}, status=web.HTTPBadRequest.status_code + ) + + LOG.info(f"Generating metadata for task '{task.name}' in '{save_path!s}'") + + from yt_dlp.utils import sanitize_filename + + from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP + + title: str = sanitize_filename(info.get("title")) + info_file: Path = save_path / f"{title} [{info.get('id')}].info.json" + info_file.write_text(encoder.encode(metadata), encoding="utf-8") + info["json_file"] = str(info_file.relative_to(config.download_path)) + + xml_file: Path = save_path / "tvshow.nfo" + info["nfo_file"] = str(xml_file.relative_to(config.download_path)) + + xml_content = "\n" + xml_content += f" {NFOMakerPP._escape_text(info.get('title'))}\n" + if info.get("description"): + xml_content += ( + f" {NFOMakerPP._escape_text(NFOMakerPP._clean_description(info.get('description')))}\n" + ) + if info.get("id"): + xml_content += f" {NFOMakerPP._escape_text(info.get('id'))}\n" + if info.get("id_type") and info.get("id"): + xml_content += f' {NFOMakerPP._escape_text(info.get("id"))}\n' + if info.get("uploader"): + xml_content += f" {NFOMakerPP._escape_text(info.get('uploader'))}\n" + if info.get("tags", []): + for tag in info.get("tags", []): + xml_content += f" {NFOMakerPP._escape_text(tag)}\n" + if info.get("year"): + xml_content += f" {info.get('year')}\n" + xml_content += " Continuing\n" + xml_content += "\n" + xml_file.write_text(xml_content, encoding="utf-8") + + try: + from yt_dlp.utils.networking import random_user_agent + + from app.library.httpx_client import async_client + + ytdlp_args: dict = task.get_ytdlp_opts().get_all() + opts: dict[str, Any] = { + "headers": { + "User-Agent": request.headers.get("User-Agent", ytdlp_args.get("user_agent", random_user_agent())), + }, + } + if proxy := ytdlp_args.get("proxy"): + opts["proxy"] = proxy + + try: + from httpx_curl_cffi import AsyncCurlTransport, CurlOpt + + opts["transport"] = AsyncCurlTransport( + impersonate="chrome", + default_headers=True, + curl_options={CurlOpt.FRESH_CONNECT: True}, + ) + opts.pop("headers", None) + except Exception: + pass + + async with async_client(**opts) as client: + for key in info.get("thumbnails", {}): + try: + url = info["thumbnails"][key] + LOG.info(f"Fetching thumbnail '{key}' from '{url}'") + if not url: + continue + + try: + validate_url(url, allow_internal=config.allow_internal_urls) + except ValueError: + LOG.warning(f"Invalid thumbnail url '{url}'") + continue + + resp = await client.request(method="GET", url=url, follow_redirects=True) + + img_file = save_path / f"{key}.jpg" + img_file.write_bytes(resp.content) + info["thumbnails"][key] = str(img_file.relative_to(config.download_path)) + except Exception as e: + LOG.warning(f"Failed to fetch thumbnail '{key}' from '{url}'. '{e!s}'") + continue + except Exception as e: + LOG.warning(f"Failed to fetch thumbnails. '{e!s}'") + + return web.json_response(data=info, status=web.HTTPOk.status_code, dumps=encoder.encode) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) diff --git a/app/features/tasks/schemas.py b/app/features/tasks/schemas.py new file mode 100644 index 00000000..a1b1fb0d --- /dev/null +++ b/app/features/tasks/schemas.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from datetime import datetime # noqa: TC003 +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from app.features.core.schemas import Pagination + + +class Task(BaseModel): + model_config = ConfigDict(from_attributes=True, str_strip_whitespace=True) + + id: int | None = None + name: str = Field(min_length=1) + url: str = Field(min_length=1) + folder: str = "" + preset: str = "" + timer: str = "" + template: str = "" + cli: str = "" + auto_start: bool = True + handler_enabled: bool = True + enabled: bool = True + created_at: datetime | None = None + updated_at: datetime | None = None + + @field_validator("name", mode="before") + @classmethod + def _normalize_name(cls, value: Any) -> str: + if not isinstance(value, str): + msg: str = "Name must be a string." + raise ValueError(msg) + value = value.strip() + if not value: + msg = "Name cannot be empty." + raise ValueError(msg) + return value + + @field_validator("url", mode="before") + @classmethod + def _normalize_url(cls, value: Any) -> str: + if not isinstance(value, str): + msg: str = "URL must be a string." + raise ValueError(msg) + value = value.strip() + if not value: + msg = "URL cannot be empty." + raise ValueError(msg) + + from app.library.Utils import validate_url + + try: + validate_url(value, allow_internal=True) + except ValueError as e: + msg = f"Invalid URL format: {e!s}" + raise ValueError(msg) from e + + return value + + @field_validator("timer", mode="before") + @classmethod + def _validate_timer(cls, value: Any) -> str: + if not value: + return "" + + if not isinstance(value, str): + msg: str = "Timer must be a string." + raise ValueError(msg) + + value = value.strip() + if not value: + return "" + + from datetime import UTC, datetime + + try: + from cronsim import CronSim + + CronSim(value, datetime.now(UTC)) + except Exception as e: + msg = f"Invalid timer format: {e!s}" + raise ValueError(msg) from e + + return value + + @field_validator("cli", mode="before") + @classmethod + def _validate_cli(cls, value: Any) -> str: + if not value: + return "" + + if not isinstance(value, str): + msg: str = "CLI must be a string." + raise ValueError(msg) + + value = value.strip() + if not value: + return "" + + from app.library.Utils import arg_converter + + try: + arg_converter(args=value) + except Exception as e: + msg = f"Invalid command options for yt-dlp: {e!s}" + raise ValueError(msg) from e + + return value + + +class TaskPatch(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + + name: str | None = None + url: str | None = None + folder: str | None = None + preset: str | None = None + timer: str | None = None + template: str | None = None + cli: str | None = None + auto_start: bool | None = None + handler_enabled: bool | None = None + enabled: bool | None = None + + +class TaskList(BaseModel): + model_config = ConfigDict(from_attributes=True) + + items: list[Task] = Field(default_factory=list) + pagination: Pagination diff --git a/app/features/tasks/service.py b/app/features/tasks/service.py new file mode 100644 index 00000000..2e4900d7 --- /dev/null +++ b/app/features/tasks/service.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent +from app.features.tasks.models import TaskModel +from app.features.tasks.utils import cron_time +from app.library.Events import Event, EventBus, Events +from app.library.Scheduler import Scheduler +from app.library.Services import Services +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from aiohttp import web + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Tasks(metaclass=Singleton): + def __init__(self): + from app.features.tasks.deps import get_tasks_repo + + self._repo = get_tasks_repo() + self._loaded: bool = False + self._handlers_service = None + self._scheduler = Scheduler.get_instance() + + @staticmethod + def get_instance() -> Tasks: + """Get the singleton instance of Tasks.""" + return Tasks() + + def attach(self, _: web.Application) -> None: + async def handle_started(_, __): + await self._repo.run_migrations() + await self._load_tasks() + await self._init_handlers_service(self._scheduler) + + Services.get_instance().add("tasks_service", self).add("tasks_repository", self._repo) + + async def handle_config_update(e: Event, _): + if isinstance(e.data, ConfigEvent) and CEFeature.TASKS == e.data.feature: + await self._handle_task_change(e.data) + + EventBus.get_instance().subscribe( + Events.CONFIG_UPDATE, handle_config_update, "Tasks.config_update_scheduler" + ).subscribe(Events.STARTED, handle_started, "TasksRepository.run_migrations") + + async def on_shutdown(self, _: web.Application) -> None: + pass + + async def _load_tasks(self) -> None: + tasks = await self._repo.list() + + for task in tasks: + if not task.timer or not task.enabled: + continue + + try: + self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=f"task-cronjob-{task.id}") + LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.") + except Exception as e: + LOG.exception(e) + LOG.error(f"Failed to queue task '{task.name}'. '{e!s}'.") + + async def _init_handlers_service(self, scheduler) -> None: + """Initialize the handlers service after migrations.""" + if self._handlers_service is not None: + return + + from app.features.tasks.definitions.service import TaskHandle + from app.library.config import Config + + config = Config.get_instance() + self._handlers_service = TaskHandle(scheduler, self._repo, config) + self._handlers_service.load() + LOG.debug("Task handlers service initialized.") + Services.get_instance().add("task_handle_service", self._handlers_service) + + async def _handle_task_change(self, event_data) -> None: + task_data: dict = event_data.data + task_id: str = f"task-cronjob-{task_data['id']}" + + if CEAction.DELETE == event_data.action: + if self._scheduler.has(task_id): + self._scheduler.remove(task_id) + + elif event_data.action in (CEAction.CREATE, CEAction.UPDATE): + if not (task := await self._repo.get(int(task_data["id"]))): + return + + if self._scheduler.has(task_id): + self._scheduler.remove(task_id) + + if task.timer and task.enabled: + self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=task_id) + LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.") + + async def _runner(self, task: TaskModel) -> None: + """ + Execute a scheduled task. + + Args: + task: The TaskModel to execute. + + """ + import time + from datetime import UTC, datetime + + from app.library.config import Config + from app.library.downloads import DownloadQueue + from app.library.ItemDTO import Item + + timeNow: str = datetime.now(UTC).isoformat() + try: + if not (task := await self._repo.get(task.id)): + LOG.info(f"Task '{task.name}' no longer exists.") + return + + if not task.enabled: + LOG.debug(f"Task '{task.name}' is disabled. Skipping execution.") + return + + if not task.url: + LOG.error(f"Failed to dispatch '{task.name}'. No URL found.") + return + + started: float = time.time() + + config = Config.get_instance() + preset: str = task.preset or config.default_preset + folder: str = task.folder or "" + template: str = task.template or "" + cli: str = task.cli or "" + + notify: EventBus = EventBus.get_instance() + + status = await DownloadQueue.get_instance().add( + item=Item.format( + { + "url": task.url, + "preset": preset, + "folder": folder, + "template": template, + "cli": cli, + "auto_start": task.auto_start, + "extras": { + "source_name": task.name, + "source_id": str(task.id), + "source_handler": "Tasks", + }, + } + ) + ) + + timeNow = datetime.now(UTC).isoformat() + ended: float = time.time() + LOG.info(f"Task '{task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.") + + notify.emit( + Events.TASK_DISPATCHED, + data={**status, "preset": task.preset} if status else {"preset": task.preset}, + title=f"Task '{task.name}' dispatched", + message=f"Task '{task.name}' dispatched at '{timeNow}'.", + ) + notify.emit( + Events.LOG_SUCCESS, + data={"preset": task.preset, "lowPriority": True}, + title="Task completed", + message=f"Task '{task.name}' completed in '{ended - started:.2f}'.", + ) + except Exception as e: + LOG.error(f"Failed to execute '{task.name}' at '{timeNow}'. '{e!s}'.") + EventBus.get_instance().emit( + Events.LOG_ERROR, + data={"preset": task.preset}, + title="Task failed", + message=f"Failed to execute '{task.name}'. '{e!s}'", + ) + + @property + def handlers(self): + """Get the handlers service instance.""" + return self._handlers_service diff --git a/app/features/tasks/tests/__init__.py b/app/features/tasks/tests/__init__.py new file mode 100644 index 00000000..4c4f8811 --- /dev/null +++ b/app/features/tasks/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for tasks feature.""" diff --git a/app/features/tasks/tests/test_tasks_repository.py b/app/features/tasks/tests/test_tasks_repository.py new file mode 100644 index 00000000..6ceacbe6 --- /dev/null +++ b/app/features/tasks/tests/test_tasks_repository.py @@ -0,0 +1,289 @@ +"""Tests for TasksRepository.""" + +from __future__ import annotations + +import pytest +import pytest_asyncio + +from app.features.tasks.models import TaskModel +from app.features.tasks.repository import TasksRepository +from app.library.sqlite_store import SqliteStore + + +@pytest_asyncio.fixture +async def repo(tmp_path): + """Provide a fresh repository instance with initialized database for each test.""" + TasksRepository._reset_singleton() + SqliteStore._reset_singleton() + + store = SqliteStore(db_path=str(":memory:")) + await store.get_connection() + + repository = TasksRepository.get_instance() + + yield repository + + if store._conn: + await store._conn.close() + if store._engine: + await store._engine.dispose() + + TasksRepository._reset_singleton() + SqliteStore._reset_singleton() + + +class TestTasksRepository: + """Test suite for TasksRepository database operations.""" + + @pytest.mark.asyncio + async def test_repository_singleton(self, repo): + """Verify repository follows singleton pattern.""" + instance1 = TasksRepository.get_instance() + instance2 = TasksRepository.get_instance() + assert instance1 is instance2, "Should return same singleton instance" + + @pytest.mark.asyncio + async def test_list_empty(self, repo): + """List returns empty when no tasks exist.""" + tasks = await repo.list() + assert tasks == [], "Should return empty list when no tasks" + + @pytest.mark.asyncio + async def test_count_empty(self, repo): + """Count returns 0 when no tasks exist.""" + count = await repo.count() + assert count == 0, "Should return 0 when no tasks exist" + + @pytest.mark.asyncio + async def test_create_task(self, repo): + """Create task with valid data.""" + data = { + "name": "Daily Download", + "url": "https://example.com/video", + "folder": "/downloads", + "preset": "audio", + "timer": "0 0 * * *", + "template": "%(title)s.%(ext)s", + "cli": "--format best", + "auto_start": True, + "handler_enabled": True, + "enabled": True, + } + + model = await repo.create(data) + + assert model.id is not None, "Should generate ID for new task" + assert model.name == "Daily Download", "Should store name correctly" + assert model.url == "https://example.com/video", "Should store URL correctly" + assert model.folder == "/downloads", "Should store folder correctly" + assert model.preset == "audio", "Should store preset correctly" + assert model.timer == "0 0 * * *", "Should store timer correctly" + assert model.template == "%(title)s.%(ext)s", "Should store template correctly" + assert model.cli == "--format best", "Should store CLI correctly" + assert model.auto_start is True, "Should store auto_start correctly" + assert model.handler_enabled is True, "Should store handler_enabled correctly" + assert model.enabled is True, "Should store enabled correctly" + assert model.created_at is not None, "Should have created_at timestamp" + assert model.updated_at is not None, "Should have updated_at timestamp" + + @pytest.mark.asyncio + async def test_create_with_minimal_data(self, repo): + """Create task with minimal required data.""" + data = { + "name": "Simple Task", + "url": "https://example.com", + } + + model = await repo.create(data) + + assert model.name == "Simple Task", "Should store name" + assert model.url == "https://example.com", "Should store URL" + assert model.folder == "", "Should default folder to empty string" + assert model.preset == "", "Should default preset to empty string" + assert model.timer == "", "Should default timer to empty string" + assert model.template == "", "Should default template to empty string" + assert model.cli == "", "Should default CLI to empty string" + assert model.auto_start is True, "Should default auto_start to True" + assert model.handler_enabled is True, "Should default handler_enabled to True" + assert model.enabled is True, "Should default enabled to True" + + @pytest.mark.asyncio + async def test_get_by_id(self, repo): + """Get task by integer ID.""" + created = await repo.create( + { + "name": "Get Test", + "url": "https://example.com", + } + ) + + retrieved = await repo.get(created.id) + + assert retrieved is not None, "Should retrieve created task" + assert retrieved.id == created.id, "Should retrieve correct task by ID" + assert retrieved.name == "Get Test", "Should match created task name" + + @pytest.mark.asyncio + async def test_get_by_string_id(self, repo): + """Get task by string ID.""" + created = await repo.create( + { + "name": "String ID Test", + "url": "https://example.com", + } + ) + + retrieved = await repo.get(str(created.id)) + + assert retrieved is not None, "Should retrieve by string ID" + assert retrieved.id == created.id, "Should match created task ID" + + @pytest.mark.asyncio + async def test_get_nonexistent(self, repo): + """Get nonexistent task returns None.""" + result = await repo.get(99999) + assert result is None, "Should return None for nonexistent ID" + + result = await repo.get("99999") + assert result is None, "Should return None for nonexistent string ID" + + @pytest.mark.asyncio + async def test_update_task(self, repo): + """Update existing task.""" + created = await repo.create( + { + "name": "Update Test", + "url": "https://example.com", + "preset": "video", + "enabled": True, + } + ) + + updated = await repo.update( + created.id, + { + "name": "Updated Name", + "preset": "audio", + "enabled": False, + }, + ) + + assert updated.name == "Updated Name", "Should update name" + assert updated.preset == "audio", "Should update preset" + assert updated.enabled is False, "Should update enabled" + assert updated.url == "https://example.com", "Should preserve unchanged URL" + + @pytest.mark.asyncio + async def test_update_nonexistent_raises(self, repo): + """Update nonexistent task raises KeyError.""" + with pytest.raises(KeyError, match="not found"): + await repo.update(99999, {"name": "should_fail"}) + + @pytest.mark.asyncio + async def test_delete_task(self, repo): + """Delete existing task.""" + created = await repo.create( + { + "name": "Delete Test", + "url": "https://example.com", + } + ) + + deleted = await repo.delete(created.id) + + assert deleted.id == created.id, "Should return deleted task" + + result = await repo.get(created.id) + assert result is None, "Deleted task should not be retrievable" + + @pytest.mark.asyncio + async def test_delete_nonexistent_raises(self, repo): + """Delete nonexistent task raises KeyError.""" + with pytest.raises(KeyError, match="not found"): + await repo.delete(99999) + + @pytest.mark.asyncio + async def test_get_by_name(self, repo): + """Get task by name.""" + await repo.create( + { + "name": "Named Task", + "url": "https://example.com", + } + ) + + retrieved = await repo.get_by_name("Named Task") + + assert retrieved is not None, "Should retrieve by name" + assert retrieved.name == "Named Task", "Should match task name" + + @pytest.mark.asyncio + async def test_get_by_name_excludes_id(self, repo): + """Get by name can exclude specific ID.""" + first = await repo.create( + { + "name": "duplicate", + "url": "https://example.com", + } + ) + + result = await repo.get_by_name("duplicate", exclude_id=first.id) + assert result is None, "Should not find when excluding only match" + + result = await repo.get_by_name("duplicate", exclude_id=None) + assert result is not None, "Should find without exclusion" + + @pytest.mark.asyncio + async def test_get_all_enabled(self, repo): + """Get all enabled tasks.""" + await repo.create({"name": "Enabled 1", "url": "https://example.com", "enabled": True}) + await repo.create({"name": "Disabled", "url": "https://example.com", "enabled": False}) + await repo.create({"name": "Enabled 2", "url": "https://example.com", "enabled": True}) + + enabled = await repo.get_all_enabled() + + assert len(enabled) == 2, "Should return only enabled tasks" + assert all(task.enabled for task in enabled), "All tasks should be enabled" + + @pytest.mark.asyncio + async def test_get_all_with_timer(self, repo): + """Get all tasks with timer configured.""" + await repo.create({"name": "With Timer", "url": "https://example.com", "timer": "0 0 * * *"}) + await repo.create({"name": "No Timer", "url": "https://example.com", "timer": ""}) + await repo.create({"name": "Another Timer", "url": "https://example.com", "timer": "0 12 * * *"}) + + with_timer = await repo.get_all_with_timer() + + assert len(with_timer) == 2, "Should return only tasks with timer" + assert all(task.timer for task in with_timer), "All tasks should have timer" + + @pytest.mark.asyncio + async def test_list_paginated(self, repo): + """List paginated returns correct subset.""" + for i in range(5): + await repo.create( + { + "name": f"Task {i}", + "url": "https://example.com", + } + ) + + items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2) + + assert len(items) == 2, "Should return 2 items per page" + assert total == 5, "Should report total count of 5" + assert page == 1, "Should be on page 1" + assert total_pages == 3, "Should have 3 pages total" + + @pytest.mark.asyncio + async def test_list_ordering(self, repo): + """List orders by name ascending.""" + await repo.create({"name": "Charlie", "url": "https://example.com"}) + await repo.create({"name": "Alice", "url": "https://example.com"}) + await repo.create({"name": "Bob", "url": "https://example.com"}) + + items = await repo.list() + + assert items[0].name == "Alice", "Should be alphabetically first" + assert items[1].name == "Bob", "Should be alphabetically second" + assert items[2].name == "Charlie", "Should be alphabetically third" diff --git a/app/features/tasks/utils.py b/app/features/tasks/utils.py new file mode 100644 index 00000000..f411a576 --- /dev/null +++ b/app/features/tasks/utils.py @@ -0,0 +1,16 @@ +import logging + +LOG: logging.Logger = logging.getLogger(__name__) + + +def cron_time(timer: str) -> str: + try: + from datetime import UTC, datetime + + from cronsim import CronSim + + cs = CronSim(timer, datetime.now(UTC)) + return cs.explain() + except Exception as exc: + LOG.exception(exc) + return timer diff --git a/app/library/Scheduler.py b/app/library/Scheduler.py index adcf3dd6..7d09b1ef 100644 --- a/app/library/Scheduler.py +++ b/app/library/Scheduler.py @@ -133,7 +133,7 @@ class Scheduler(metaclass=Singleton): def remove(self, id: str | list[str]) -> bool: """ - Remove a job from the schedule. + Remove a job from the scheduler. Args: id (str|list[str]): The id of the job to remove. @@ -156,7 +156,7 @@ class Scheduler(metaclass=Singleton): return False del self._jobs[id] - LOG.debug(f"Removed job '{id}' from the schedule.") + LOG.debug(f"Removed job '{id}' from the scheduler.") return True return False diff --git a/app/library/Services.py b/app/library/Services.py index 743fccab..eaecd529 100644 --- a/app/library/Services.py +++ b/app/library/Services.py @@ -48,7 +48,7 @@ class Services(metaclass=Singleton): def get_instance() -> "Services": return Services() - def add(self, name: str, service: Any, declared_type: type | None = None): + def add(self, name: str, service: Any, declared_type: type | None = None) -> "Services": """ Add a service by name. @@ -57,12 +57,16 @@ class Services(metaclass=Singleton): service: The service instance. declared_type: The declared type of the service (optional). + Returns: + Services: The Services instance (for chaining). + """ if declared_type is None and service is not None: declared_type = type(service) self.remove(name) self._services.append(ServiceEntry(name=name, declared_type=declared_type, instance=service)) + return self def add_all(self, services: dict[str, Any]): for name, svc in services.items(): diff --git a/app/library/TaskDefinitions.py b/app/library/TaskDefinitions.py deleted file mode 100644 index 189aa347..00000000 --- a/app/library/TaskDefinitions.py +++ /dev/null @@ -1,435 +0,0 @@ -import json -import logging -import uuid -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from aiohttp import web -from jsonschema import Draft7Validator, SchemaError, ValidationError - -from .config import Config -from .encoder import Encoder -from .Services import Services -from .Singleton import Singleton - -LOG: logging.Logger = logging.getLogger("task_definitions") - - -@dataclass(slots=True) -class TaskDefinitionRecord: - identifier: str - """UUID identifier of the task definition.""" - filename: str - """Filename of the task definition JSON file.""" - name: str - """Human-readable name of the task definition.""" - priority: int - """Priority of the task definition.""" - path: Path - """Path to the task definition JSON file.""" - data: dict[str, Any] - """The task definition data.""" - updated_at: float - """Last modified timestamp of the task definition file.""" - - def serialize(self, *, include_definition: bool = False) -> dict[str, Any]: - """ - Serialize the task definition record to a dictionary. - - Args: - include_definition (bool): Whether to include the full task definition data. - - Returns: - dict[str, Any]: The serialized task definition record. - - """ - payload: dict[str, Any] = { - "id": self.identifier, - "name": self.name, - "priority": self.priority, - "updated_at": self.updated_at, - } - - if include_definition: - payload["definition"] = self.data - - return payload - - def json(self, *, include_definition: bool = False) -> str: - """ - Serialize the task definition record to a JSON string. - - Args: - include_definition (bool): Whether to include the full task definition data. - - Returns: - str: The JSON string representation of the task definition record. - - """ - return Encoder().encode(self.serialize(include_definition=include_definition)) - - -class TaskDefinitions(metaclass=Singleton): - def __init__( - self, - directory: str | Path | None = None, - config: Config | None = None, - validator: Draft7Validator | None = None, - ): - self._config: Config = config or Config.get_instance() - "Instance of Config to use." - self._directory: Path = Path(directory) if directory else Path(self._config.config_path) / "tasks" - "Directory where task definition files are stored." - self._validator: Draft7Validator | None = validator - "JSON schema validator instance." - self._items: dict[str, TaskDefinitionRecord] = {} - "Mapping of task definition ID to TaskDefinitionRecord." - self._schema: Path = Path(self._config.app_path) / "schema" / "task_definition.json" - "Path to the JSON schema file for task definitions." - - try: - self._directory.mkdir(parents=True, exist_ok=True) - except Exception as e: - LOG.error(f"Failed to create tasks directory '{self._directory}': {e}") - - self.load() - - @staticmethod - def get_instance( - directory: str | Path | None = None, - config: Config | None = None, - validator: Draft7Validator | None = None, - ) -> "TaskDefinitions": - """ - Get the singleton instance of TaskDefinitions. - - Args: - directory (str | Path | None): Optional directory to store task definitions. - config (Config | None): Optional Config instance to use. - validator (Draft7Validator | None): Optional JSON schema validator to use. - - Returns: - TaskDefinitions: The singleton instance of TaskDefinitions. - - """ - return TaskDefinitions(directory=directory, config=config, validator=validator) - - def attach(self, _: web.Application) -> None: - """ - Attach the TaskDefinitions service to the application. - - Args: - _ (web.Application): The aiohttp web application instance. - - """ - Services.get_instance().add("task_definitions", self) - - async def on_shutdown(self, _: web.Application) -> None: - """ - Handle application shutdown event. - - Args: - _ (web.Application): The aiohttp web application instance. - - """ - return - - def _get_validator(self) -> Draft7Validator: - """ - Get or create the JSON schema validator for task definitions. - - Returns: - Draft7Validator: The JSON schema validator instance. - - """ - if self._validator: - return self._validator - - try: - contents: str = self._schema.read_text(encoding="utf-8") - schema = json.loads(contents) - except Exception as e: - LOG.error(f"Failed to read task definition schema '{self._schema}': {e}") - raise - - try: - self._validator = Draft7Validator(schema) - except SchemaError as e: - LOG.error(f"Invalid task definition schema '{self._schema}': {e}") - raise - - return self._validator - - def validate(self, definition: dict[str, Any]) -> None: - """ - Validate a task definition against the JSON schema. - - Args: - definition (dict[str, Any]): The task definition to validate. - - Raises: - ValueError: If the task definition is invalid. - - """ - try: - self._get_validator().validate(definition) - except ValidationError as e: - path: str = " ".join(str(part) for part in e.path) - error_path: str = f" ({path})" if path else "" - message: str = f"Task definition validation failed{error_path}: {e.message}" - raise ValueError(message) from e - - def load(self) -> "TaskDefinitions": - """ - Load all task definitions from the directory. - - Returns: - TaskDefinitions: The current instance for chaining. - - """ - self._items.clear() - - if not self._directory.exists(): - return self - - for file_path in sorted(self._directory.glob("*.json")): - stem: str = file_path.stem - - try: - identifier_uuid = uuid.UUID(stem) - except ValueError: - LOG.warning(f"Skipping task definition with invalid UUID filename '{file_path.name}'.") - continue - - if 4 != identifier_uuid.version: - LOG.warning(f"Skipping task definition '{file_path.name}', Name is not UUIDv4.") - continue - - try: - contents: str = file_path.read_text(encoding="utf-8") - except Exception as e: - LOG.error(f"Failed to load task definition '{file_path}': {e!s}") - continue - - try: - parsed = json.loads(contents) - except Exception as e: - LOG.error(f"Failed to parse task definition '{file_path}': {e!s}") - continue - - if not isinstance(parsed, dict): - LOG.error(f"Invalid task definition file '{file_path}': must be a JSON object.") - continue - - data: dict[str, Any] = parsed - - identifier: str = str(identifier_uuid) - - name_value: str = str(data.get("name") or identifier) - priority: int = self._normalize_priority(data.get("priority", 0)) - data["priority"] = priority - - record = TaskDefinitionRecord( - identifier=identifier, - filename=file_path.name, - name=name_value, - priority=priority, - path=file_path, - data=data, - updated_at=file_path.stat().st_mtime, - ) - self._items[record.identifier] = record - - return self - - def list(self) -> list[TaskDefinitionRecord]: - """ - List all task definitions, sorted by priority and name. - - Returns: - list[TaskDefinitionRecord]: List of task definitions sorted by priority and name. - - """ - return sorted( - self._items.values(), - key=lambda record: (record.priority, record.name.lower()), - ) - - def get(self, identifier: str) -> TaskDefinitionRecord | None: - """ - Get a task definition by its identifier. - - Args: - identifier (str): The UUID identifier of the task definition. - - Returns: - TaskDefinitionRecord | None: The task definition record, or None if not found. - - """ - return self._items.get(identifier) - - def _path_for(self, identifier: str) -> Path: - """ - Get the file path for a given task definition identifier. - - Args: - identifier (str): The UUID identifier of the task definition. - - Returns: - Path: The file path for the task definition JSON file. - - """ - return self._directory / f"{identifier}.json" - - def _write_file(self, path: Path, payload: dict[str, Any]) -> None: - """ - Write a task definition to a file. - - Args: - path (Path): The file path to write to. - payload (dict[str, Any]): The task definition data to write. - - Raises: - Exception: If writing to the file fails. - - """ - path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") - try: - path.chmod(0o600) - except Exception: - pass - - def _normalize_priority(self, value: Any) -> int: - """ - Normalize the priority value to an integer. - - Args: - value (Any): The priority value to normalize. - - Returns: - int: The normalized priority value. - - """ - try: - priority = int(value) - except Exception: - priority = 0 - - return priority - - def _refresh_generic_handler(self) -> None: - """ - Refresh the generic task handler definitions. - """ - try: - from .task_handlers.generic import GenericTaskHandler - - GenericTaskHandler.refresh_definitions(force=True) - except Exception as e: - LOG.error(f"Failed to refresh generic task handler: {e}") - - def create(self, definition: dict[str, Any]) -> TaskDefinitionRecord: - """ - Create a new task definition. - - Args: - definition (dict[str, Any]): The task definition data. - - Returns: - TaskDefinitionRecord: The created task definition record. - - """ - self.validate(definition) - - identifier: str = str(uuid.uuid4()) - path: Path = self._path_for(identifier) - - while path.exists(): - identifier = str(uuid.uuid4()) - path = self._path_for(identifier) - - priority: int = self._normalize_priority(definition.get("priority", 0)) - definition["priority"] = priority - - self._write_file(path, definition) - - record = TaskDefinitionRecord( - identifier=identifier, - filename=path.name, - name=str(definition.get("name", identifier)), - priority=priority, - path=path, - data=definition, - updated_at=path.stat().st_mtime, - ) - - self._items[identifier] = record - self._refresh_generic_handler() - return record - - def update(self, identifier: str, definition: dict[str, Any]) -> TaskDefinitionRecord: - """ - Update an existing task definition. - - Args: - identifier (str): The UUID identifier of the task definition to update. - definition (dict[str, Any]): The updated task definition data. - - Returns: - TaskDefinitionRecord: The updated task definition record. - - Raises: - ValueError: If the task definition does not exist or is invalid. - - """ - record: TaskDefinitionRecord | None = self.get(identifier) - if not record: - message: str = f"Task definition '{identifier}' does not exist." - raise ValueError(message) - - self.validate(definition) - priority: int = self._normalize_priority(definition.get("priority", record.priority)) - definition["priority"] = priority - self._write_file(record.path, definition) - - updated_record = TaskDefinitionRecord( - identifier=identifier, - filename=record.filename, - name=str(definition.get("name", identifier)), - priority=priority, - path=record.path, - data=definition, - updated_at=record.path.stat().st_mtime, - ) - - self._items[identifier] = updated_record - self._refresh_generic_handler() - return updated_record - - def delete(self, identifier: str) -> None: - """ - Delete a task definition. - - Args: - identifier (str): The UUID identifier of the task definition to delete. - - Raises: - ValueError: If the task definition does not exist. - - """ - record: TaskDefinitionRecord | None = self.get(identifier) - if not record: - message: str = f"Task definition '{identifier}' does not exist." - raise ValueError(message) - - try: - record.path.unlink(missing_ok=False) - except FileNotFoundError: - LOG.warning(f"Task definition file '{record.path}' already removed.") - except Exception as exc: - LOG.error(f"Failed to delete task definition '{identifier}': {exc}") - raise - - self._items.pop(identifier, None) - self._refresh_generic_handler() diff --git a/app/library/Tasks.py b/app/library/Tasks.py deleted file mode 100644 index ad4689e6..00000000 --- a/app/library/Tasks.py +++ /dev/null @@ -1,1072 +0,0 @@ -import asyncio -import inspect -import json -import logging -import pkgutil -import random -import time -import uuid -from dataclasses import asdict, dataclass, field -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from aiohttp import web - -from .config import Config -from .downloads import DownloadQueue -from .encoder import Encoder -from .Events import EventBus, Events -from .ItemDTO import Item, ItemDTO -from .Scheduler import Scheduler -from .Services import Services -from .Singleton import Singleton -from .Utils import archive_add, archive_delete, archive_read, fetch_info, init_class, validate_url -from .YTDLPOpts import YTDLPOpts - -LOG: logging.Logger = logging.getLogger("tasks") - - -@dataclass(kw_only=True) -class Task: - id: str - name: str - url: str - folder: str = "" - preset: str = "" - timer: str = "" - template: str = "" - cli: str = "" - auto_start: bool = True - handler_enabled: bool = True - enabled: bool = True - - def serialize(self) -> dict: - return self.__dict__ - - def json(self) -> str: - return Encoder().encode(self.serialize()) - - def get(self, key: str, default: Any = None) -> Any: - """ - Get a value from the task by key. - - Args: - key (str): The key to get. - default (Any): The default value if the key is not found. - - Returns: - Any: The value of the key or the default value. - - """ - return self.serialize().get(key, default) - - def get_ytdlp_opts(self) -> YTDLPOpts: - """ - Get the yt-dlp options for the task. - - Returns: - YTDLPOpts: The yt-dlp options. - - """ - params: YTDLPOpts = YTDLPOpts.get_instance() - - if self.preset: - params = params.preset(name=self.preset) - - if self.cli: - params = params.add_cli(self.cli, from_user=True) - - if self.template: - params = params.add({"outtmpl": {"default": self.template}}, from_user=False) - - return params - - async def mark(self) -> tuple[bool, str]: - """ - Mark the task's items as downloaded in the archive file. - - Returns: - tuple[bool, str]: A tuple indicating success and a message. - - """ - ret: tuple[bool, str] | set[tuple[Path, set[str]]] = await self._mark_logic() - if isinstance(ret, tuple): - return ret - - archive_file: Path = ret.get("file") - items: set[str] = ret.get("items", set()) - - if len(items) < 1 or not archive_add(archive_file, list(items)): - return (True, "No new items to mark as downloaded.") - - return (True, f"Task '{self.name}' items marked as downloaded.") - - async def unmark(self) -> tuple[bool, str]: - """ - Unmark the task's items from the archive file. - - Returns: - tuple[bool, str]: A tuple indicating success and a message. - - """ - ret: tuple[bool, str] | set[tuple[Path, set[str]]] = await self._mark_logic() - if isinstance(ret, tuple): - return ret - - archive_file: Path = ret.get("file") - items: set[str] = ret.get("items", set()) - - if len(items) < 1 or not archive_delete(archive_file, list(items)): - return (True, "No items to remove from archive file.") - - return (True, f"Removed '{self.name}' items from archive file.") - - async def fetch_metadata(self, full: bool = False) -> tuple[dict[str, Any] | None, bool, str]: - """ - Fetch metadata for the task's URL. - - Args: - full (bool): Whether to fetch full metadata including all entries for playlists. - - Returns: - tuple[dict[str, Any]|None, bool, str]: A tuple containing the metadata (or None on failure), a boolean - indicating if the operation was successful, and a message. - - """ - if not self.url: - return ({}, False, "No URL found in task parameters.") - - params = self.get_ytdlp_opts() - if not full: - params.add_cli("-I0", from_user=False) - - params = params.get_all() - - ie_info: dict | None = await fetch_info( - params, - self.url, - no_archive=True, - follow_redirect=False, - sanitize_info=True, - ) - - if not ie_info or not isinstance(ie_info, dict): - return ({}, False, "Failed to extract information from URL.") - - return (ie_info, True, "") - - async def _mark_logic(self) -> tuple[bool, str] | set[tuple[Path, set[str]]]: - if not self.url: - return (False, "No URL found in task parameters.") - - params: dict = self.get_ytdlp_opts().get_all() - if not (archive_file := params.get("download_archive")): - return (False, "No archive file found.") - - archive_file: Path = Path(archive_file) - - ie_info: dict | None = await fetch_info(params, self.url, no_archive=True, follow_redirect=True) - if not ie_info or not isinstance(ie_info, dict): - return (False, "Failed to extract information from URL.") - - if "playlist" != ie_info.get("_type"): - return (False, "Expected a playlist type from extract_info.") - - items: set[str] = set() - - def _process(item: dict): - for entry in item.get("entries", []): - if not isinstance(entry, dict): - continue - - if "playlist" == entry.get("_type"): - _process(entry) - continue - - if entry.get("_type") not in ("video", "url"): - continue - - if not entry.get("id") or not entry.get("ie_key"): - continue - - archive_id: str = f"{entry.get('ie_key', '').lower()} {entry.get('id')}" - - items.add(archive_id) - - _process(ie_info) - - return {"file": archive_file, "items": items} - - -def _split_inspect_metadata(metadata: dict[str, Any] | None) -> tuple[dict[str, Any], dict[str, Any]]: - """ - Split commonly consumed metadata keys from the rest. - - Args: - metadata (dict[str, Any]|None): The metadata to split. - - Returns: - tuple[dict[str, Any], dict[str, Any]]: The primary and extra metadata. - - """ - metadata = dict(metadata or {}) - primary: dict[str, Any] = {} - - for key in ("matched", "handler", "supported"): - if key in metadata: - primary[key] = metadata.pop(key) - - return primary, metadata - - -@dataclass(slots=True) -class TaskItem: - url: str - "The URL of the item." - title: str | None = None - "The title of the item." - archive_id: str | None = None - "The archive ID of the item." - metadata: dict[str, Any] = field(default_factory=dict) - "Additional metadata related to the item." - - -@dataclass(slots=True) -class TaskResult: - items: list[TaskItem] = field(default_factory=list) - "The list of items." - metadata: dict[str, Any] = field(default_factory=dict) - "Additional metadata related to the result." - - def serialize(self) -> dict[str, Any]: - """ - Serialize the task result. - - Returns: - dict[str, Any]: The serialized task result. - - """ - primary, extra = _split_inspect_metadata(self.metadata) - payload: dict[str, Any] = {**primary, "items": [asdict(item) for item in self.items]} - - if extra: - payload["metadata"] = extra - - return payload - - -@dataclass(slots=True) -class TaskFailure: - message: str - "A human-readable message describing the failure." - error: str | None = None - "An optional error code or string." - metadata: dict[str, Any] = field(default_factory=dict) - "Additional metadata related to the failure." - - def serialize(self) -> dict[str, Any]: - """ - Serialize the task failure. - - Returns: - dict[str, Any]: The serialized task failure. - - """ - primary, extra = _split_inspect_metadata(self.metadata) - payload: dict[str, Any] = dict(primary) - - if self.error: - payload["error"] = self.error - - if self.message and (not self.error or self.message != self.error): - payload["message"] = self.message - - if extra: - payload["metadata"] = extra - - return payload - - -class Tasks(metaclass=Singleton): - """ - This class is used to manage the tasks. - """ - - def __init__( - self, - file: str | None = None, - loop: asyncio.AbstractEventLoop | None = None, - config: Config | None = None, - encoder: Encoder | None = None, - scheduler: Scheduler | None = None, - ): - self._tasks: list[Task] = [] - "The tasks." - config = config or Config.get_instance() - - self._debug: bool = config.debug - "Debug mode." - self._default_preset: str = config.default_preset - "The default preset." - self._file: Path = Path(file) if file else Path(config.config_path).joinpath("tasks.json") - "The tasks file." - self._encoder: Encoder = encoder or Encoder() - "The JSON encoder." - self._loop: asyncio.AbstractEventLoop = loop or asyncio.get_event_loop() - "The event loop." - self._scheduler: Scheduler = scheduler or Scheduler.get_instance() - "The scheduler." - self._notify: EventBus = EventBus.get_instance() - "The event bus." - self._task_handler = HandleTask(self._scheduler, self, config) - "The task handler." - self._downloadQueue = DownloadQueue.get_instance() - "The download queue." - - if self._file.exists() and "600" != self._file.stat().st_mode: - try: - self._file.chmod(0o600) - except Exception: - pass - - @staticmethod - def get_instance( - file: str | None = None, - loop: asyncio.AbstractEventLoop | None = None, - config: Config | None = None, - encoder: Encoder | None = None, - scheduler: Scheduler | None = None, - ) -> "Tasks": - """ - Get the instance of the class. - - Returns: - Tasks: The instance of the class. - - """ - return Tasks(file=file, loop=loop, config=config, encoder=encoder, scheduler=scheduler) - - async def on_shutdown(self, _: web.Application): - self.clear(shutdown=True) - self._task_handler.on_shutdown(_) - - def attach(self, _: web.Application): - """ - Attach the tasks to the aiohttp application. - - Args: - _ (web.Application): The aiohttp application. - - """ - self.load() - Services.get_instance().add("tasks", self) - - async def event_handler(data, _): - if data and data.data: - self.save(data.data) - - self._notify.subscribe(Events.TASKS_ADD, event_handler, f"{__class__.__name__}.add") - self._task_handler.load() - - def get_all(self) -> list[Task]: - """Return the tasks.""" - return self._tasks - - def get_handler(self) -> "HandleTask": - """Expose the handle task helper.""" - return self._task_handler - - def get(self, task_id: str) -> Task | None: - """ - Get a task by its ID. - - Args: - task_id (str): The ID of the task. - - Returns: - Task | None: The task if found, otherwise None. - - """ - return next((task for task in self._tasks if task.id == task_id), None) - - def load(self) -> "Tasks": - """ - Load the tasks. - - Returns: - Tasks: The current instance. - - """ - has_tasks: bool = len(self._tasks) > 0 - self.clear() - - if not self._file.exists() or self._file.stat().st_size < 1: - return self - - try: - LOG.info(f"{'Reloading' if has_tasks else 'Loading'} '{self._file}'.") - tasks = json.loads(self._file.read_text()) - except Exception as e: - LOG.error(f"Error loading '{self._file}'. '{e!s}'.") - return self - - if not tasks or len(tasks) < 1: - return self - - needs_update: bool = False - for i, task in enumerate(tasks): - try: - Tasks.validate(task) - if "enabled" not in task: - task["enabled"] = True - needs_update = True - - task: Task = init_class(Task, task) - except Exception as e: - LOG.error(f"Failed to parse task at list position '{i}'. '{e!s}'.") - continue - - self._tasks.append(task) - - if not task.timer: - continue - - try: - self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=task.id) - - try: - from cronsim import CronSim - - cs = CronSim(task.timer, datetime.now(UTC)) - schedule_time: str = cs.explain() - except Exception: - schedule_time = task.timer - - if not has_tasks: - LOG.info(f"Task '{task.name}' queued to be executed '{schedule_time}'.") - except Exception as e: - LOG.exception(e) - LOG.error(f"Failed to queue '{i}: {task.name}'. '{e!s}'.") - - if needs_update: - self.save(self._tasks) - - return self - - def clear(self, shutdown: bool = False) -> "Tasks": - """ - Clear all tasks. - - Returns: - Tasks: The current instance. - - """ - if len(self._tasks) < 1: - return self - - for task in self._tasks: - if not self._scheduler.has(task.id): - continue - - try: - LOG.debug(f"Stopping '{task.name}'.") - self._scheduler.remove(task.id) - except Exception as e: - if not shutdown: - LOG.exception(e) - LOG.error(f"Failed to stop '{task.name}'. '{e!s}'.") - - self._tasks.clear() - - return self - - @staticmethod - def validate(task: Task | dict) -> bool: - """ - Validate the task. - - Args: - task (Task|dict): The task to validate. - - Returns: - bool: True if the task is valid, False otherwise. - - """ - if not isinstance(task, dict): - if not isinstance(task, Task): - msg = "Invalid task type." - raise ValueError(msg) - - task = task.serialize() - - if not task.get("name"): - msg = "No name found." - raise ValueError(msg) - - task["name"] = task["name"].strip() - - if not task.get("url"): - msg = "No URL found." - raise ValueError(msg) - - task["url"] = task["url"].strip() - try: - validate_url(task["url"], allow_internal=True) - except ValueError as e: - msg = f"Invalid URL format. '{e!s}'." - raise ValueError(msg) from e - - if task.get("timer"): - try: - from cronsim import CronSim - - CronSim(task.get("timer"), datetime.now(UTC)) - task["timer"] = str(task["timer"]).strip() - except Exception as e: - msg = f"Invalid timer format. '{e!s}'." - raise ValueError(msg) from e - - if task.get("cli"): - try: - from .Utils import arg_converter - - arg_converter(args=task.get("cli")) - task["cli"] = str(task["cli"]).strip() - except Exception as e: - msg = f"Invalid command options for yt-dlp. '{e!s}'." - raise ValueError(msg) from e - - return True - - def save(self, tasks: list[Task | dict]) -> "Tasks": - """ - Save the tasks. - - Args: - tasks (list[Task]): The tasks to save. - - Returns: - Tasks: The current instance. - - """ - for i, task in enumerate(tasks): - try: - self.validate(task) - - if not isinstance(task, Task): - task: Task = init_class(Task, task) - tasks[i] = task - except ValueError as e: - LOG.error(f"Failed to validate item '{i}: {task.name}'. '{e}'.") - continue - except Exception as e: - LOG.error(f"Failed to save task '{i}'. '{e!s}'.") - continue - - try: - self._file.write_text(json.dumps([i.serialize() for i in tasks], indent=4)) - LOG.info(f"Updated '{self._file}'.") - except Exception as e: - LOG.error(f"Error saving '{self._file}'. '{e!s}'.") - - return self - - async def _runner(self, task: Task) -> None: - """ - Run the task. - - Args: - task (Task): The task to run. - - Returns: - None - - """ - timeNow: str = datetime.now(UTC).isoformat() - try: - if not self.get(task.id): - LOG.info(f"Task '{task.name}' no longer exists.") - if self._scheduler.has(task.id): - self._scheduler.remove(task.id) - return - - started: float = time.time() - - if not task.enabled: - LOG.debug(f"Task '{task.name}' is disabled. Skipping execution.") - return - - if not task.url: - LOG.error(f"Failed to dispatch '{task.name}'. No URL found.") - return - - preset: str = str(task.preset or self._default_preset) - folder: str = task.folder if task.folder else "" - template: str = task.template if task.template else "" - cli: str = task.cli if task.cli else "" - - status = await self._downloadQueue.add( - item=Item.format( - { - "url": task.url, - "preset": preset, - "folder": folder, - "template": template, - "cli": cli, - "auto_start": task.auto_start, - "extras": { - "source_name": task.name, - "source_id": task.id, - "source_handler": __class__.__name__, - }, - } - ) - ) - - timeNow = datetime.now(UTC).isoformat() - ended: float = time.time() - LOG.info(f"Task '{task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.") - self._notify.emit( - Events.TASK_DISPATCHED, - data={**status, "preset": task.preset} if status else {"preset": task.preset}, - title=f"Task '{task.name}' dispatched", - message=f"Task '{task.name}' dispatched at '{timeNow}'.", - ) - self._notify.emit( - Events.LOG_SUCCESS, - data={"preset": task.preset, "lowPriority": True}, - title="Task completed", - message=f"Task '{task.name}' completed in '{ended - started:.2f}'.", - ) - except Exception as e: - LOG.error(f"Failed to execute '{task.name}' at '{timeNow}'. '{e!s}'.") - self._notify.emit( - Events.LOG_ERROR, - data={"preset": task.preset}, - title="Task failed", - message=f"Failed to execute '{task.name}'. '{e!s}'", - ) - - -class HandleTask: - def __init__(self, scheduler: Scheduler, tasks: Tasks, config: Config) -> None: - self._handlers: list[type] = [] - "The available handlers." - self._tasks: Tasks = tasks - "The tasks manager." - self._scheduler: Scheduler = scheduler - "The scheduler." - self._config: Config = config - "The configuration." - self._task_name: str = f"{__class__.__name__}._dispatcher" - "The task name for the scheduler." - self._queued: dict[str, set[str]] = {} - "Queued archive IDs per handler." - self._failure_count: dict[str, dict[str, int]] = {} - "Failure counts per handler and archive ID." - - EventBus.get_instance().subscribe( - Events.ITEM_ERROR, - self._handle_item_error, - f"{__class__.__name__}.item_error", - ) - - def load(self) -> None: - """ - Load the available handlers and schedule the dispatcher. - """ - self._handlers: list[type] = self._discover() - - timer: str = self._config.tasks_handler_timer - try: - from cronsim import CronSim - - CronSim(timer, datetime.now(UTC)) - except Exception as e: - timer = "15 */1 * * *" - LOG.error(f"Invalid timer format. '{e!s}'. Defaulting to '{timer}'.") - - self._scheduler.add( - timer=timer, - func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"), - id=f"{__class__.__name__}._dispatcher", - ) - - def on_shutdown(self, _: web.Application) -> None: - """ - Handle shutdown event. - - Args: - _: web.Application: The aiohttp application. - - """ - if self._scheduler.has(self._task_name): - self._scheduler.remove(self._task_name) - - async def _dispatcher(self): - s: dict[list[str]] = {"h": [], "d": [], "u": [], "f": []} - - handler_groups: dict[str, list[tuple[Task, type]]] = {} - - for task in self._tasks.get_all(): - if not task.enabled or not task.handler_enabled: - s["d"].append(task.name) - continue - - if not task.get_ytdlp_opts().get_all().get("download_archive"): - LOG.debug(f"Task '{task.name}' does not have an archive file configured.") - s["f"].append(task.name) - continue - - try: - handler = await self._find_handler(task) - if handler is None: - s["u"].append(task.name) - continue - - handler_name = handler.__name__ - if handler_name not in handler_groups: - handler_groups[handler_name] = [] - handler_groups[handler_name].append((task, handler)) - s["h"].append(task.name) - except Exception as e: - LOG.error(f"Failed to handle task '{task.name}'. '{e!s}'.") - s["f"].append(task.name) - - for tasks_with_handlers in handler_groups.values(): - for idx, (task, handler) in enumerate(tasks_with_handlers): - try: - t = asyncio.create_task( - coro=self._dispatch( - task, - handler, - delay=0.0 if 0 == idx else random.uniform(1.0, self._config.task_handler_random_delay), - ), - name=f"task-{task.id}", - ) - t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t)) - except Exception as e: - LOG.error(f"Failed to dispatch task '{task.name}'. '{e!s}'.") - - if len(self._tasks.get_all()) > 0: - LOG.info( - f"Tasks handler summary: Handled: {len(s['h'])}, Unhandled: {len(s['u'])}, Disabled: {len(s['d'])}, Failed: {len(s['f'])}." - ) - - async def _dispatch(self, task: Task, handler: type, delay: float) -> TaskResult | TaskFailure | None: - """ - Dispatch a task after a random delay to avoid rate limiting. - - Args: - task (Task): The task to dispatch. - handler (type): The handler to use. - delay (float): The delay in seconds before dispatching. - - Returns: - TaskResult|TaskFailure|None: The dispatch result. - - """ - if delay > 0: - LOG.debug(f"Delaying dispatch of task '{task.name}' by {delay:.1f} seconds.") - await asyncio.sleep(delay) - return await self.dispatch(task, handler=handler) - - def _handle_exception(self, fut: asyncio.Task, task: Task) -> None: - if fut.cancelled(): - return - - if exc := fut.exception(): - LOG.error(f"Exception while handling task '{task.name}': {exc}") - - async def _find_handler(self, task: Task) -> type | None: - for cls in self._handlers: - try: - if await Services.get_instance().handle_async(handler=cls.can_handle, task=task): - return cls - except Exception as e: - LOG.exception(e) - continue - - return None - - async def dispatch(self, task: Task, handler: type | None = None, **kwargs) -> TaskResult | TaskFailure | None: # noqa: ARG002 - """ - Dispatch a task to the appropriate handler. - - Args: - task (Task): The task to dispatch. - handler (type|None): Optional specific handler to use instead of finding one. - **kwargs: Additional context to pass to the handler. - - Returns: - TaskResult|TaskFailure|None: The extraction outcome, or None if no handler matched. - - """ - if not handler: - handler = await self._find_handler(task) - if handler is None: - return None - - services: Services = Services.get_instance() - - try: - extraction: TaskResult | TaskFailure = await services.handle_async( - handler=handler.extract, task=task, config=self._config - ) - except NotImplementedError: - LOG.error(f"Handler '{handler.__name__}' does not implement extract().") - return TaskFailure(message="Handler does not support extraction.") - except Exception as exc: - LOG.exception(exc) - raise - - if isinstance(extraction, TaskFailure): - LOG.error(f"Handler '{handler.__name__}' failed to extract items: {extraction.message}") - return extraction - - if not isinstance(extraction, TaskResult): - LOG.error( - f"Handler '{handler.__name__}' returned unexpected result type '{type(extraction).__name__}'.", - ) - return TaskFailure( - message="Handler returned invalid result type.", metadata={"type": type(extraction).__name__} - ) - - raw_items: list[TaskItem] = extraction.items or [] - metadata: dict[str, Any] = extraction.metadata or {} - - handler_name: str = handler.__name__ - queued: set[str] = self._queued.setdefault(handler_name, set()) - failures: dict[str, int] = self._failure_count.setdefault(handler_name, {}) - - params: dict = task.get_ytdlp_opts().get_all() - archive_file: str | None = params.get("download_archive") - - download_queue: DownloadQueue = services.get("queue") or DownloadQueue.get_instance() - notify: EventBus = services.get("notify") or EventBus.get_instance() - - archive_ids: list[str] = [ - item.archive_id for item in raw_items if isinstance(item, TaskItem) and item.archive_id - ] - downloaded: list[str] = archive_read(archive_file, archive_ids) if archive_file else [] - - filtered: list[TaskItem] = [] - - for item in raw_items: - if not isinstance(item, TaskItem): - LOG.warning("Handler '{handler.__name__}' produced non-TaskItem entry: {item!r}") - continue - - url: str = item.url - if not url: - continue - - archive_id: str | None = item.archive_id - if not archive_id: - LOG.warning(f"'{task.name}': Item with URL '{url}' is missing an archive ID. Skipping.") - continue - - if archive_id in queued: - continue - - queued.add(archive_id) - - if archive_file and archive_id in downloaded: - continue - - if await download_queue.queue.exists(url=url): - continue - - try: - done = await download_queue.done.get(url=url) - if "error" != done.info.status: - continue - except KeyError: - pass - - if archive_id not in failures: - failures[archive_id] = 0 - - filtered.append(item) - - if not filtered: - if raw_items: - LOG.debug( - f"Handler '{handler.__name__}' produced '{len(raw_items)}' for '{task.name}' items, none queued after filtering." - ) - return TaskResult(items=[], metadata=metadata) - - LOG.info( - f"Handler '{handler.__name__}' Found '{len(filtered)}' new items for '{task.name}' (raw={len(raw_items)})." - ) - - base_item = Item.format( - { - "url": task.url, - "preset": task.preset or self._config.default_preset, - "folder": task.folder or "", - "template": task.template or "", - "cli": task.cli or "", - "auto_start": task.auto_start, - "extras": {"source_name": task.name, "source_id": task.id, "source_handler": handler.__name__}, - } - ) - - for item in filtered: - metadata_entry: dict[str, Any] = item.metadata if isinstance(item.metadata, dict) else {} - extras: dict[str, Any] = base_item.extras.copy() - if metadata_entry: - extras["metadata"] = metadata_entry - - notify.emit( - Events.ADD_URL, - data=base_item.new_with(url=item.url, extras=extras).serialize(), - ) - - return TaskResult(items=filtered, metadata=metadata) - - async def inspect( - self, - url: str, - preset: str | None = None, - handler_name: str | None = None, - static_only: bool = False, - ) -> TaskResult | TaskFailure: - if not self._handlers: - self._handlers = self._discover() - - task = Task( - id=str(uuid.uuid4()), - name="Inspector", - url=url, - preset=preset or self._config.default_preset, - auto_start=False, - ) - - services = Services.get_instance() - - handler_cls: type | None - if handler_name: - handler_cls = next((cls for cls in self._handlers if cls.__name__.lower() == handler_name.lower()), None) - if handler_cls is None: - message: str = f"Handler '{handler_name}' not found." - return TaskFailure( - message=message, - error=message, - metadata={"matched": False, "handler": handler_name}, - ) - - try: - matched = await services.handle_async(handler=handler_cls.can_handle, task=task) - except Exception as exc: # pragma: no cover - defensive - LOG.exception(exc) - message = str(exc) - return TaskFailure( - message=message, - error=message, - metadata={"matched": False, "handler": handler_cls.__name__}, - ) - - if not matched: - return TaskFailure( - message="Handler cannot process the supplied URL.", - metadata={"matched": False, "handler": handler_cls.__name__}, - ) - else: - handler_cls = await self._find_handler(task) - if handler_cls is None: - message = "No handler matched the supplied URL." - return TaskFailure( - message=message, - error=message, - metadata={"matched": False, "handler": None}, - ) - - base_metadata: dict[str, Any] = {"matched": True, "handler": handler_cls.__name__} - - if static_only: - return TaskResult(items=[], metadata=base_metadata) - - try: - extraction: TaskResult | TaskFailure = await services.handle_async( - handler=handler_cls.extract, task=task, config=self._config - ) - except NotImplementedError: - return TaskFailure( - message="Handler does not support manual inspection.", - metadata={**base_metadata, "supported": False}, - ) - except Exception as exc: - LOG.exception(exc) - message = str(exc) - return TaskFailure( - message=message, - error=message, - metadata={**base_metadata, "supported": True}, - ) - - if isinstance(extraction, TaskFailure): - combined_failure_metadata: dict[str, Any] = {**base_metadata, "supported": True} - if extraction.metadata: - combined_failure_metadata.update(extraction.metadata) - - return TaskFailure( - message=extraction.message, - error=extraction.error if extraction.error else extraction.message, - metadata=combined_failure_metadata, - ) - - if not isinstance(extraction, TaskResult): - LOG.error( - f"Handler '{handler_cls.__name__}' returned unexpected result type '{type(extraction).__name__}' during inspection.", - ) - extraction = TaskResult() - - combined_metadata: dict[str, Any] = {**base_metadata, "supported": True} - if extraction.metadata: - combined_metadata.update(extraction.metadata) - - return TaskResult(items=list(extraction.items), metadata=combined_metadata) - - def _discover(self) -> list[type]: - import importlib - - import app.features.tasks.definitions.handlers as handlers_pkg - - handlers: list[type] = [] - - for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__): - if module_name.startswith("_"): - continue - - module = importlib.import_module(f"{handlers_pkg.__name__}.{module_name}") - for _, cls in inspect.getmembers(module, inspect.isclass): - if cls.__module__ != module.__name__: - continue - - if callable(getattr(cls, "can_handle", None)) and callable(getattr(cls, "extract", None)): - handlers.append(cls) - - return handlers - - async def _handle_item_error(self, event, _name, **_kwargs): - item: ItemDTO | None = getattr(event, "data", None) - if not isinstance(item, ItemDTO): - return - - extras: dict[Any, Any] = getattr(item, "extras", {}) or {} - handler_name: Any | None = extras.get("source_handler") - if not handler_name: - return - - archive_id: str | None = item.archive_id - if not archive_id: - return - - queued: set[str] | None = self._queued.get(handler_name) - if queued: - queued.discard(archive_id) - - failures: dict[str, int] = self._failure_count.setdefault(handler_name, {}) - failures[archive_id] = failures.get(archive_id, 0) + 1 diff --git a/app/main.py b/app/main.py index c050ccd7..78a49dff 100644 --- a/app/main.py +++ b/app/main.py @@ -18,6 +18,7 @@ from app.features.conditions.service import Conditions from app.features.dl_fields.service import DLFields from app.features.notifications.service import Notifications from app.features.tasks.definitions.deps import get_task_definitions_repo +from app.features.tasks.service import Tasks from app.library.BackgroundWorker import BackgroundWorker from app.library.cache import Cache from app.library.config import Config @@ -29,7 +30,6 @@ from app.library.Presets import Presets from app.library.Scheduler import Scheduler from app.library.Services import Services from app.library.sqlite_store import SqliteStore -from app.library.Tasks import Tasks from app.library.UpdateChecker import UpdateChecker LOG = logging.getLogger("app") diff --git a/app/migrations/20260124171740_add_tasks_table.py b/app/migrations/20260124171740_add_tasks_table.py new file mode 100644 index 00000000..7ff56c81 --- /dev/null +++ b/app/migrations/20260124171740_add_tasks_table.py @@ -0,0 +1,46 @@ +""" +This module contains a db migration. + +Migration Name: add_tasks_table +Migration Version: 20260124171740 +""" + +from sqlalchemy import text + + +async def upgrade(c): + sql: list[str] = [ + """ + CREATE TABLE IF NOT EXISTS tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + url TEXT NOT NULL, + folder TEXT NOT NULL DEFAULT '', + preset TEXT NOT NULL DEFAULT '', + timer TEXT NOT NULL DEFAULT '', + template TEXT NOT NULL DEFAULT '', + cli TEXT NOT NULL DEFAULT '', + auto_start INTEGER NOT NULL DEFAULT 1, + handler_enabled INTEGER NOT NULL DEFAULT 1, + enabled INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """, + 'CREATE INDEX IF NOT EXISTS "ix_tasks_name" ON "tasks" ("name")', + 'CREATE INDEX IF NOT EXISTS "ix_tasks_enabled" ON "tasks" ("enabled")', + 'CREATE INDEX IF NOT EXISTS "ix_tasks_timer" ON "tasks" ("timer")', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) + + +async def downgrade(c): + sql: list[str] = [ + 'DROP INDEX IF EXISTS "ix_tasks_name"', + 'DROP INDEX IF EXISTS "ix_tasks_enabled"', + 'DROP INDEX IF EXISTS "ix_tasks_timer"', + 'DROP TABLE IF EXISTS "tasks"', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) diff --git a/app/routes/api/tasks.py b/app/routes/api/tasks.py index 9ae1beac..30a548c2 100644 --- a/app/routes/api/tasks.py +++ b/app/routes/api/tasks.py @@ -1,459 +1,3 @@ -import logging -import uuid -from pathlib import Path -from typing import TYPE_CHECKING, Any +"""Migrated API routes for feature submodule.""" -from aiohttp import web -from aiohttp.web import Request, Response - -from app.library.ag_utils import ag -from app.library.config import Config -from app.library.encoder import Encoder -from app.library.router import route -from app.library.Tasks import Task, TaskFailure, TaskResult, Tasks -from app.library.Utils import get_channel_images, get_file, init_class, parse_outtmpl, validate_url, validate_uuid - -if TYPE_CHECKING: - from pathlib import Path - -LOG: logging.Logger = logging.getLogger(__name__) - - -@route("POST", "api/tasks/inspect", "task_handler_inspect") -async def task_handler_inspect(request: Request, tasks: Tasks, encoder: Encoder, config: Config) -> Response: - """ - Check if handler can process the given URL. - - Args: - request (Request): The request object. - tasks (Tasks): The tasks instance. - encoder (Encoder): The encoder instance. - config (Config): The config instance. - - Returns: - Response: The response object. - - """ - data = await request.json() - - url: str | None = data.get("url") if isinstance(data, dict) else None - if not url: - return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code) - - static_only: bool = data.get("static_only", False) if isinstance(data, dict) else False - if not static_only: - try: - validate_url(url, allow_internal=config.allow_internal_urls) - except ValueError as e: - return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code) - - preset: str = data.get("preset", "") if isinstance(data, dict) else "" - handler_name: str | None = data.get("handler") if isinstance(data, dict) else None - - try: - result: TaskResult | TaskFailure = await tasks.get_handler().inspect( - url=url, preset=preset, handler_name=handler_name, static_only=static_only - ) - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to inspect handler.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - return web.json_response( - data=result, - status=web.HTTPBadRequest.status_code if isinstance(result, TaskFailure) else web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("GET", "api/tasks/", "tasks") -async def tasks(encoder: Encoder) -> Response: - """ - Get the tasks. - - Args: - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object. - - """ - return web.json_response(data=Tasks.get_instance().get_all(), status=web.HTTPOk.status_code, dumps=encoder.encode) - - -@route("PUT", "api/tasks/", "tasks_add") -async def tasks_add(request: Request, encoder: Encoder) -> Response: - """ - Add tasks to the queue. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - data = await request.json() - - if not isinstance(data, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - tasks: list = [] - - ins = Tasks.get_instance() - - for item in data: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("url"): - return web.json_response({"error": "url is required.", "data": item}, status=web.HTTPBadRequest.status_code) - - if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - if not item.get("template", None): - item["template"] = "" - - if not item.get("cli", None): - item["cli"] = "" - - try: - Tasks.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Failed to validate task '{item.get('name', '??')}'. '{e!s}'"}, - status=web.HTTPBadRequest.status_code, - ) - - tasks.append(init_class(Task, item)) - - try: - tasks = ins.save(tasks=tasks).load().get_all() - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to save tasks.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - return web.json_response(data=tasks, status=web.HTTPOk.status_code, dumps=encoder.encode) - - -@route("POST", "api/tasks/{id}/mark", "tasks_mark") -async def task_mark(request: Request, encoder: Encoder) -> Response: - """ - Mark all items from task as downloaded. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - task_id: str = request.match_info.get("id", None) - - if not task_id: - return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) - - tasks: Tasks = Tasks.get_instance() - try: - task: Task | None = tasks.get(task_id) - if not task: - return web.json_response( - data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code - ) - - _status, _message = await task.mark() - if not _status: - return web.json_response(data={"error": _message}, status=web.HTTPBadRequest.status_code) - - return web.json_response(data={"message": _message}, status=web.HTTPOk.status_code, dumps=encoder.encode) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - - -@route("DELETE", "api/tasks/{id}/mark", "tasks_unmark") -async def task_unmark(request: Request, encoder: Encoder) -> Response: - """ - Remove All tasks items from download archive. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - task_id: str = request.match_info.get("id", None) - - if not task_id: - return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) - - tasks: Tasks = Tasks.get_instance() - try: - task: Task | None = tasks.get(task_id) - if not task: - return web.json_response( - data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code - ) - - _status, _message = await task.unmark() - if not _status: - return web.json_response(data={"error": _message}, status=web.HTTPBadRequest.status_code) - - return web.json_response(data={"message": _message}, status=web.HTTPOk.status_code, dumps=encoder.encode) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - - -@route("POST", "api/tasks/{id}/metadata", "tasks_metadata") -async def task_metadata(request: Request, config: Config, encoder: Encoder) -> Response: - """ - Generate metadata for the task. - - Args: - request (Request): The request object. - config (Config): The config instance. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - task_id: str = request.match_info.get("id", None) - - if not task_id: - return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) - - tasks: Tasks = Tasks.get_instance() - try: - task: Task | None = tasks.get(task_id) - if not task: - return web.json_response( - data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code - ) - - (save_path, _) = get_file(config.download_path, task.folder) - if not str(save_path or "").startswith(str(config.download_path)): - return web.json_response(data={"error": "Invalid task folder."}, status=web.HTTPBadRequest.status_code) - - if not save_path.exists(): - save_path.mkdir(parents=True, exist_ok=True) - - metadata, status, message = await task.fetch_metadata() - if not status: - return web.json_response(data={"error": message}, status=web.HTTPBadRequest.status_code) - - if not task.folder: - try: - ytdlp_opts: dict = task.get_ytdlp_opts().get_all() - outtmpl: str = parse_outtmpl( - output_template=ytdlp_opts.get("outtmpl", {}).get("default", "{title} [{id}]"), - info_dict=metadata, - params=ytdlp_opts, - ) - if outtmpl: - _path: Path = save_path / outtmpl - if not _path.is_dir(): - _path: Path = _path.parent - - (save_path, _) = get_file(config.download_path, _path.relative_to(config.download_path)) - if not str(save_path or "").startswith(str(config.download_path)): - return web.json_response( - data={"error": "Invalid final path folder."}, status=web.HTTPBadRequest.status_code - ) - - if not save_path.exists(): - save_path.mkdir(parents=True, exist_ok=True) - except Exception as e: - LOG.warning(f"Failed to resolve final path from outtmpl. '{e!s}'") - - info = { - "id": ag(metadata, ["id", "channel_id"]), - "id_type": metadata.get("extractor", "").split(":")[0].lower() if metadata.get("extractor") else None, - "title": ag(metadata, ["title", "fulltitle"]) or None, - "description": metadata.get("description", ""), - "uploader": metadata.get("uploader", ""), - "tags": metadata.get("tags", []), - "year": metadata.get("release_year"), - "thumbnails": get_channel_images(metadata.get("thumbnails", {})), - } - - if not info.get("title"): - return web.json_response( - data={"error": "Failed to get title from metadata."}, status=web.HTTPBadRequest.status_code - ) - - LOG.info(f"Generating metadata for task '{task.name}' in '{save_path!s}'") - - from yt_dlp.utils import sanitize_filename - - from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP - - title: str = sanitize_filename(info.get("title")) - info_file: Path = save_path / f"{title} [{info.get('id')}].info.json" - info_file.write_text(encoder.encode(metadata), encoding="utf-8") - info["json_file"] = str(info_file.relative_to(config.download_path)) - - xml_file: Path = save_path / "tvshow.nfo" - info["nfo_file"] = str(xml_file.relative_to(config.download_path)) - - xml_content = "\n" - xml_content += f" {NFOMakerPP._escape_text(info.get('title'))}\n" - if info.get("description"): - xml_content += ( - f" {NFOMakerPP._escape_text(NFOMakerPP._clean_description(info.get('description')))}\n" - ) - if info.get("id"): - xml_content += f" {NFOMakerPP._escape_text(info.get('id'))}\n" - if info.get("id_type") and info.get("id"): - xml_content += f' {NFOMakerPP._escape_text(info.get("id"))}\n' - if info.get("uploader"): - xml_content += f" {NFOMakerPP._escape_text(info.get('uploader'))}\n" - if info.get("tags", []): - for tag in info.get("tags", []): - xml_content += f" {NFOMakerPP._escape_text(tag)}\n" - if info.get("year"): - xml_content += f" {info.get('year')}\n" - xml_content += " Continuing\n" - xml_content += "\n" - xml_file.write_text(xml_content, encoding="utf-8") - - try: - from yt_dlp.utils.networking import random_user_agent - - from app.library.httpx_client import async_client - - ytdlp_args: dict = task.get_ytdlp_opts().get_all() - opts: dict[str, Any] = { - "headers": { - "User-Agent": request.headers.get("User-Agent", ytdlp_args.get("user_agent", random_user_agent())), - }, - } - if proxy := ytdlp_args.get("proxy"): - opts["proxy"] = proxy - - try: - from httpx_curl_cffi import AsyncCurlTransport, CurlOpt - - opts["transport"] = AsyncCurlTransport( - impersonate="chrome", - default_headers=True, - curl_options={CurlOpt.FRESH_CONNECT: True}, - ) - opts.pop("headers", None) - except Exception: - pass - - async with async_client(**opts) as client: - for key in info.get("thumbnails", {}): - try: - url = info["thumbnails"][key] - LOG.info(f"Fetching thumbnail '{key}' from '{url}'") - if not url: - continue - - try: - validate_url(url, allow_internal=config.allow_internal_urls) - except ValueError: - LOG.warning(f"Invalid thumbnail url '{url}'") - continue - - resp = await client.request(method="GET", url=url, follow_redirects=True) - - img_file = save_path / f"{key}.jpg" - img_file.write_bytes(resp.content) - info["thumbnails"][key] = str(img_file.relative_to(config.download_path)) - except Exception as e: - LOG.warning(f"Failed to fetch thumbnail '{key}' from '{url}'. '{e!s}'") - continue - except Exception as e: - LOG.warning(f"Failed to fetch thumbnails. '{e!s}'") - - return web.json_response(data=info, status=web.HTTPOk.status_code, dumps=encoder.encode) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - - -@route("PATCH", "api/tasks/{id}", "tasks_update") -async def task_update(request: Request, encoder: Encoder) -> Response: - """ - Update specific fields of a task. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - if not (task_id := request.match_info.get("id", None)): - return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) - - try: - data = await request.json() - except Exception as e: - return web.json_response( - data={"error": "Invalid JSON in request body.", "message": str(e)}, - status=web.HTTPBadRequest.status_code, - ) - - if not isinstance(data, dict): - return web.json_response( - {"error": "Request body must be a JSON object."}, - status=web.HTTPBadRequest.status_code, - ) - - tasks: Tasks = Tasks.get_instance() - try: - task: Task | None = tasks.get(task_id) - if not task: - return web.json_response( - data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code - ) - - task_dict: dict = task.serialize() - protected_fields: set[str] = {"id"} - updated: bool = False - - for key, value in data.items(): - if key in protected_fields or key not in task_dict: - continue - - setattr(task, key, value) - updated = True - - if not updated: - return web.json_response( - data={"error": "No valid fields to update."}, - status=web.HTTPBadRequest.status_code, - ) - - try: - Tasks.validate(task) - except ValueError as e: - return web.json_response({"error": f"Validation failed: {e!s}"}, status=web.HTTPBadRequest.status_code) - - tasks.save(tasks=tasks.get_all()).load() - - return web.json_response(data=task, status=web.HTTPOk.status_code, dumps=encoder.encode) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": "Failed to update task.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) +import app.features.tasks.router # noqa: F401 diff --git a/app/tests/test_tasks.py b/app/tests/test_tasks.py deleted file mode 100644 index 5d3dcb55..00000000 --- a/app/tests/test_tasks.py +++ /dev/null @@ -1,1183 +0,0 @@ -import tempfile -from pathlib import Path -from unittest.mock import AsyncMock, Mock, patch - -import pytest - -from app.library.Tasks import Task, Tasks - - -class TestTask: - """Test the Task dataclass.""" - - def test_task_creation_with_required_fields(self): - """Test creating a task with only required fields.""" - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - assert task.id == "test_id" - assert task.name == "test_task" - assert task.url == "https://example.com/video" - assert task.folder == "" - assert task.preset == "" - assert task.timer == "" - assert task.template == "" - assert task.cli == "" - assert task.auto_start is True - assert task.handler_enabled is True - assert task.enabled is True - - def test_task_creation_with_all_fields(self): - """Test creating a task with all fields specified.""" - task = Task( - id="test_id", - name="test_task", - url="https://example.com/video", - folder="test_folder", - preset="test_preset", - timer="0 */6 * * *", - template="%(title)s.%(ext)s", - cli="--extract-flat", - auto_start=False, - handler_enabled=False, - enabled=False, - ) - - assert task.id == "test_id" - assert task.name == "test_task" - assert task.url == "https://example.com/video" - assert task.folder == "test_folder" - assert task.preset == "test_preset" - assert task.timer == "0 */6 * * *" - assert task.template == "%(title)s.%(ext)s" - assert task.cli == "--extract-flat" - assert task.auto_start is False - assert task.handler_enabled is False - assert task.enabled is False - - def test_task_serialize(self): - """Test task serialization to dictionary.""" - task = Task( - id="test_id", - name="test_task", - url="https://example.com/video", - preset="test_preset", - ) - - serialized = task.serialize() - - assert isinstance(serialized, dict) - assert serialized["id"] == "test_id" - assert serialized["name"] == "test_task" - assert serialized["url"] == "https://example.com/video" - assert serialized["preset"] == "test_preset" - assert serialized["folder"] == "" - assert serialized["auto_start"] is True - - @patch("app.library.Tasks.Encoder") - def test_task_json(self, mock_encoder): - """Test task JSON serialization.""" - mock_encoder_instance = Mock() - mock_encoder_instance.encode.return_value = '{"id": "test_id"}' - mock_encoder.return_value = mock_encoder_instance - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - json_str = task.json() - - assert json_str == '{"id": "test_id"}' - mock_encoder_instance.encode.assert_called_once() - - def test_task_get_method(self): - """Test task get method for accessing fields.""" - task = Task( - id="test_id", - name="test_task", - url="https://example.com/video", - preset="test_preset", - ) - - assert task.get("id") == "test_id" - assert task.get("name") == "test_task" - assert task.get("preset") == "test_preset" - assert task.get("nonexistent") is None - assert task.get("nonexistent", "default_value") == "default_value" - - @patch("app.library.Tasks.YTDLPOpts") - def test_task_get_ytdlp_opts_without_preset_or_cli(self, mock_ytdlp_opts): - """Test getting YTDLPOpts without preset or CLI options.""" - mock_opts_instance = Mock() - mock_ytdlp_opts.get_instance.return_value = mock_opts_instance - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - result = task.get_ytdlp_opts() - - assert result == mock_opts_instance - mock_ytdlp_opts.get_instance.assert_called_once() - - @patch("app.library.Tasks.YTDLPOpts") - def test_task_get_ytdlp_opts_with_preset_and_cli(self, mock_ytdlp_opts): - """Test getting YTDLPOpts with preset and CLI options.""" - mock_opts_instance = Mock() - mock_preset_opts = Mock() - mock_final_opts = Mock() - - mock_opts_instance.preset.return_value = mock_preset_opts - mock_preset_opts.add_cli.return_value = mock_final_opts - mock_ytdlp_opts.get_instance.return_value = mock_opts_instance - - task = Task( - id="test_id", - name="test_task", - url="https://example.com/video", - preset="test_preset", - cli="--extract-flat", - ) - - result = task.get_ytdlp_opts() - - assert result == mock_final_opts - mock_opts_instance.preset.assert_called_once_with(name="test_preset") - mock_preset_opts.add_cli.assert_called_once_with("--extract-flat", from_user=True) - - @pytest.mark.asyncio - @patch("app.library.Tasks.archive_add") - @patch("app.library.Tasks.fetch_info") - async def test_task_mark_success(self, mock_fetch_info, mock_archive_add): - """Test successful task marking.""" - # Mock fetch_info to return playlist data - mock_fetch_info.return_value = { - "_type": "playlist", - "entries": [ - { - "_type": "video", - "id": "video1", - "ie_key": "Youtube", - }, - { - "_type": "video", - "id": "video2", - "ie_key": "Youtube", - }, - ], - } - mock_archive_add.return_value = True - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {"download_archive": "/tmp/archive.txt"} - - task = Task(id="test_id", name="test_task", url="https://example.com/playlist") - - success, message = await task.mark() - - assert success is True - assert "marked as downloaded" in message - mock_archive_add.assert_called_once() - # Check that archive_add was called with the correct items - call_args = mock_archive_add.call_args[0] - archive_file = call_args[0] - items = call_args[1] - assert str(archive_file) == "/tmp/archive.txt" - assert "youtube video1" in items - assert "youtube video2" in items - - @pytest.mark.asyncio - async def test_task_mark_no_url(self): - """Test task marking with no URL.""" - task = Task(id="test_id", name="test_task", url="") - - success, message = await task.mark() - - assert success is False - assert "No URL found" in message - - @pytest.mark.asyncio - async def test_task_mark_no_archive_file(self): - """Test task marking with no archive file configured.""" - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {} - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - success, message = await task.mark() - - assert success is False - assert "No archive file found" in message - - @pytest.mark.asyncio - @patch("app.library.Tasks.archive_delete") - @patch("app.library.Tasks.fetch_info") - async def test_task_unmark_success(self, mock_fetch_info, mock_archive_delete): - """Test successful task unmarking.""" - # Mock fetch_info to return playlist data - mock_fetch_info.return_value = { - "_type": "playlist", - "entries": [ - { - "_type": "video", - "id": "video1", - "ie_key": "Youtube", - }, - ], - } - mock_archive_delete.return_value = True - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {"download_archive": "/tmp/archive.txt"} - - task = Task(id="test_id", name="test_task", url="https://example.com/playlist") - - success, message = await task.unmark() - - assert success is True - assert "Removed" in message - assert "items from archive file" in message - mock_archive_delete.assert_called_once() - - @pytest.mark.asyncio - @patch("app.library.Tasks.fetch_info") - async def test_task_mark_logic_invalid_extract_info(self, mock_fetch_info): - """Test _mark_logic with invalid fetch_info response.""" - mock_fetch_info.return_value = None - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {"download_archive": "/tmp/archive.txt"} - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - result = await task._mark_logic() - - assert isinstance(result, tuple) - success, message = result - assert success is False - assert "Failed to extract information" in message - - @pytest.mark.asyncio - @patch("app.library.Tasks.fetch_info") - async def test_task_mark_logic_not_playlist(self, mock_fetch_info): - """Test _mark_logic with non-playlist type.""" - mock_fetch_info.return_value = { - "_type": "video", - "id": "video1", - } - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {"download_archive": "/tmp/archive.txt"} - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - result = await task._mark_logic() - - assert isinstance(result, tuple) - success, message = result - assert success is False - assert "Expected a playlist type" in message - - @pytest.mark.asyncio - async def test_task_fetch_metadata_no_url(self): - """Test fetch_metadata with no URL.""" - task = Task(id="test_id", name="test_task", url="") - - metadata, success, message = await task.fetch_metadata() - - assert success is False, "Should return False when URL is missing" - assert metadata == {}, "Should return empty dict for metadata" - assert "No URL found" in message, "Should indicate missing URL" - - @pytest.mark.asyncio - @patch("app.library.Tasks.fetch_info") - async def test_task_fetch_metadata_success_not_full(self, mock_fetch_info): - """Test fetch_metadata without full flag.""" - mock_fetch_info.return_value = { - "_type": "playlist", - "id": "playlist123", - "title": "Test Playlist", - "entries": [{"id": "video1"}], - } - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_opts = Mock() - mock_opts.add_cli.return_value = mock_opts - mock_opts.get_all.return_value = {"download_archive": "/tmp/archive.txt"} - mock_get_opts.return_value = mock_opts - - task = Task(id="test_id", name="test_task", url="https://example.com/playlist") - - metadata, success, message = await task.fetch_metadata(full=False) - - assert success is True, "Should return True on successful fetch" - assert metadata["_type"] == "playlist", "Should return playlist metadata" - assert metadata["id"] == "playlist123", "Should return correct ID" - assert message == "", "Should have empty message on success" - mock_opts.add_cli.assert_called_once_with("-I0", from_user=False) - mock_fetch_info.assert_called_once() - call_kwargs = mock_fetch_info.call_args[1] - assert call_kwargs["no_archive"] is True, "Should disable archive" - assert call_kwargs["follow_redirect"] is False, "Should not follow redirects" - assert call_kwargs["sanitize_info"] is True, "Should sanitize info" - - @pytest.mark.asyncio - @patch("app.library.Tasks.fetch_info") - async def test_task_fetch_metadata_success_full(self, mock_fetch_info): - """Test fetch_metadata with full flag.""" - mock_fetch_info.return_value = { - "_type": "playlist", - "id": "playlist123", - "title": "Test Playlist", - "entries": [ - {"id": "video1", "title": "Video 1"}, - {"id": "video2", "title": "Video 2"}, - ], - } - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_opts = Mock() - mock_opts.get_all.return_value = {} - mock_get_opts.return_value = mock_opts - - task = Task(id="test_id", name="test_task", url="https://example.com/playlist") - - metadata, success, message = await task.fetch_metadata(full=True) - - assert success is True, "Should return True on successful fetch" - assert len(metadata["entries"]) == 2, "Should return full entries" - assert message == "", "Should have empty message on success" - mock_opts.add_cli.assert_not_called() - - @pytest.mark.asyncio - @patch("app.library.Tasks.fetch_info") - async def test_task_fetch_metadata_fetch_info_returns_none(self, mock_fetch_info): - """Test fetch_metadata when fetch_info returns None.""" - mock_fetch_info.return_value = None - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {} - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - metadata, success, message = await task.fetch_metadata() - - assert success is False, "Should return False when fetch_info fails" - assert metadata == {}, "Should return empty dict for metadata" - assert "Failed to extract information" in message, "Should indicate extraction failure" - - @pytest.mark.asyncio - @patch("app.library.Tasks.fetch_info") - async def test_task_fetch_metadata_fetch_info_returns_invalid_type(self, mock_fetch_info): - """Test fetch_metadata when fetch_info returns non-dict.""" - mock_fetch_info.return_value = "invalid" - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {} - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - metadata, success, message = await task.fetch_metadata() - - assert success is False, "Should return False when fetch_info returns invalid type" - assert metadata == {}, "Should return empty dict for metadata" - assert "Failed to extract information" in message, "Should indicate extraction failure" - - -class TestTasks: - """Test the Tasks singleton manager.""" - - def setup_method(self): - """Set up test environment before each test.""" - # Reset singleton instance to ensure clean state - Tasks._reset_singleton() - - def teardown_method(self): - """Clean up after each test.""" - # Reset singleton instance to prevent test pollution - Tasks._reset_singleton() - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_singleton(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test that Tasks follows singleton pattern.""" - mock_config.get_instance.return_value = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - instance1 = Tasks.get_instance() - instance2 = Tasks.get_instance() - - assert instance1 is instance2 - assert isinstance(instance1, Tasks) - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_initialization(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test Tasks initialization with proper dependencies.""" - mock_config_instance = Mock( - debug=True, default_preset="test_preset", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler_instance = Mock() - mock_scheduler.get_instance.return_value = mock_scheduler_instance - mock_download_queue_instance = Mock() - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "tasks.json" - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - assert tasks._debug is True, "Check initialization" - assert tasks._default_preset == "test_preset" - assert tasks._file == tasks_file - assert tasks._tasks == [] - assert tasks._scheduler == mock_scheduler_instance - assert tasks._notify == mock_eventbus_instance - assert tasks._downloadQueue == mock_download_queue_instance - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_get_all_empty(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test getting all tasks when no tasks exist.""" - mock_config.get_instance.return_value = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks.get_instance() - - result = tasks.get_all() - - assert result == [] - assert isinstance(result, list) - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_get_by_id_not_found(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test getting task by ID when task doesn't exist.""" - mock_config.get_instance.return_value = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks.get_instance() - - result = tasks.get("nonexistent_id") - - assert result is None - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_load_empty_file(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test loading tasks from empty or non-existent file.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "nonexistent.json" - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - result = tasks.load() - - assert result == tasks # Should return self - assert tasks.get_all() == [] - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_load_valid_file(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test loading tasks from valid JSON file.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler_instance = Mock() - mock_scheduler.get_instance.return_value = mock_scheduler_instance - mock_download_queue.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "tasks.json" - - # Create valid JSON file with task data - task_data = [ - { - "id": "task1", - "name": "Test Task 1", - "url": "https://example.com/video1", - "preset": "default", - "timer": "0 6 * * *", - "folder": "", - "template": "", - "cli": "", - "auto_start": True, - "handler_enabled": True, - } - ] - tasks_file.write_text("[\n" + "\n".join(f" {line}" for line in str(task_data).split("\n")) + "\n]") - - # Mock json.loads to return our test data - with patch("json.loads", return_value=task_data): - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - # Mock the scheduler.add method - mock_scheduler_instance.add = Mock() - - result = tasks.load() - - assert result == tasks - loaded_tasks = tasks.get_all() - assert len(loaded_tasks) == 1 - assert loaded_tasks[0].id == "task1" - assert loaded_tasks[0].name == "Test Task 1" - assert loaded_tasks[0].url == "https://example.com/video1" - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_load_invalid_json(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test loading tasks from file with invalid JSON.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "tasks.json" - tasks_file.write_text("invalid json {") - - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - result = tasks.load() - - assert result == tasks - assert tasks.get_all() == [] # Should be empty due to invalid JSON - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_save_valid_tasks(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test saving valid tasks to file.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "tasks.json" - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - task_data = [ - { - "id": "task1", - "name": "Test Task", - "url": "https://example.com/video", - "preset": "default", - "folder": "", - "template": "", - "cli": "", - "timer": "", - "auto_start": True, - "handler_enabled": True, - } - ] - - result = tasks.save(task_data) - - assert result == tasks - assert tasks_file.exists() - assert tasks_file.stat().st_size > 0, "Verify file content was written" - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_get_by_id_found(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test getting task by ID when task exists.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks(file=None, config=mock_config_instance) - - # Manually add a task to the internal list - test_task = Task(id="test_id", name="Test Task", url="https://example.com/video") - tasks._tasks.append(test_task) - - result = tasks.get("test_id") - - assert result is not None - assert result.id == "test_id" - assert result.name == "Test Task" - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_get_all_with_tasks(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test getting all tasks when tasks exist.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks(file=None, config=mock_config_instance) - - # Manually add tasks to the internal list - task1 = Task(id="task1", name="Task 1", url="https://example.com/video1") - task2 = Task(id="task2", name="Task 2", url="https://example.com/video2") - tasks._tasks.extend([task1, task2]) - - result = tasks.get_all() - - assert len(result) == 2 - assert result[0].id == "task1" - assert result[1].id == "task2" - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_clear_with_scheduler_cleanup(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test clearing tasks with scheduler cleanup.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler_instance = Mock() - mock_scheduler_instance.has.return_value = True - mock_scheduler_instance.remove = Mock() - mock_scheduler.get_instance.return_value = mock_scheduler_instance - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks(file=None, config=mock_config_instance) - - # Add a task with timer to the internal list - task_with_timer = Task(id="task1", name="Task 1", url="https://example.com/video1", timer="0 6 * * *") - tasks._tasks.append(task_with_timer) - - result = tasks.clear() - - assert result == tasks - assert len(tasks.get_all()) == 0 - mock_scheduler_instance.has.assert_called_with("task1") - mock_scheduler_instance.remove.assert_called_with("task1") - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_clear_no_tasks(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test clearing when no tasks exist.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler_instance = Mock() - mock_scheduler.get_instance.return_value = mock_scheduler_instance - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks(file=None, config=mock_config_instance) - - result = tasks.clear() - - assert result == tasks - assert len(tasks.get_all()) == 0 - # Should not call scheduler methods when no tasks exist - mock_scheduler_instance.has.assert_not_called() - mock_scheduler_instance.remove.assert_not_called() - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_validate_valid_task_dict(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test task validation with valid task dictionary.""" - mock_config.get_instance.return_value = Mock() - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - valid_task = { - "name": "Test Task", - "url": "https://example.com/video", - "timer": "0 6 * * *", - "cli": "--write-info-json", - } - - with patch("app.library.Tasks.validate_url") as mock_validate_url: - mock_validate_url.return_value = True - - result = Tasks.validate(valid_task) - - assert result is True - mock_validate_url.assert_called_once_with("https://example.com/video", allow_internal=True) - - def test_tasks_validate_invalid_task_type(self): - """Test task validation with invalid task type.""" - invalid_task = "not a dict or Task" - - with pytest.raises(ValueError, match="Invalid task type"): - Tasks.validate(invalid_task) - - def test_tasks_validate_missing_name(self): - """Test task validation with missing name.""" - invalid_task = { - "url": "https://example.com/video", - } - - with pytest.raises(ValueError, match="No name found"): - Tasks.validate(invalid_task) - - def test_tasks_validate_missing_url(self): - """Test task validation with missing URL.""" - invalid_task = { - "name": "Test Task", - } - - with pytest.raises(ValueError, match="No URL found"): - Tasks.validate(invalid_task) - - def test_tasks_validate_invalid_url(self): - """Test task validation with invalid URL.""" - invalid_task = { - "name": "Test Task", - "url": "invalid-url", - } - - with patch("app.library.Tasks.validate_url") as mock_validate_url: - mock_validate_url.side_effect = ValueError("Invalid URL") - - with pytest.raises(ValueError, match="Invalid URL format"): - Tasks.validate(invalid_task) - - def test_tasks_validate_invalid_timer(self): - """Test task validation with invalid timer format.""" - invalid_task = { - "name": "Test Task", - "url": "https://example.com/video", - "timer": "invalid timer", - } - - with patch("app.library.Tasks.validate_url") as mock_validate_url: - mock_validate_url.return_value = True - - # Import error will be raised by cronsim for invalid format - with patch("cronsim.CronSim") as mock_cronsim: - mock_cronsim.side_effect = Exception("Invalid cron format") - - with pytest.raises(ValueError, match="Invalid timer format"): - Tasks.validate(invalid_task) - - def test_tasks_validate_invalid_cli(self): - """Test task validation with invalid CLI options.""" - invalid_task = { - "name": "Test Task", - "url": "https://example.com/video", - "cli": "invalid --cli options", - } - - with patch("app.library.Tasks.validate_url") as mock_validate_url: - mock_validate_url.return_value = True - - with patch("app.library.Utils.arg_converter") as mock_arg_converter: - mock_arg_converter.side_effect = Exception("Invalid CLI args") - - with pytest.raises(ValueError, match="Invalid command options for yt-dlp"): - Tasks.validate(invalid_task) - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Item") - @patch("app.library.Tasks.datetime") - @patch("app.library.Tasks.time") - async def test_tasks_runner_success( - self, mock_time, mock_datetime, mock_item, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test successful task runner execution.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler.get_instance.return_value = Mock() - - # Mock download queue - mock_download_queue_instance = Mock() - mock_download_queue_instance.add = AsyncMock(return_value={"success": True, "id": "download123"}) - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - # Mock time and datetime - mock_time.time.side_effect = [1000.0, 1005.5] # start and end times - mock_datetime_instance = Mock() - mock_datetime_instance.isoformat.return_value = "2024-01-01T12:00:00Z" - mock_datetime.now.return_value = mock_datetime_instance - - # Mock Item.format - mock_item.format.return_value = {"formatted": "item"} - - tasks = Tasks(file=None, config=mock_config_instance) - - test_task = Task( - id="task1", - name="Test Task", - url="https://example.com/video", - preset="test_preset", - folder="test_folder", - template="test_template", - cli="--write-info-json", - auto_start=True, - enabled=True, - ) - - # Add task to tasks list so it can be found by _runner - tasks._tasks.append(test_task) - - await tasks._runner(test_task) - - # Verify download queue was called - mock_download_queue_instance.add.assert_called_once() - - # Verify Item.format was called with correct parameters - mock_item.format.assert_called_once_with( - { - "url": "https://example.com/video", - "preset": "test_preset", - "folder": "test_folder", - "template": "test_template", - "cli": "--write-info-json", - "auto_start": True, - "extras": { - "source_name": "Test Task", - "source_id": "task1", - "source_handler": "Tasks", - }, - } - ) - - assert mock_eventbus_instance.emit.call_count == 2, "Verify events were emitted" - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.datetime") - async def test_tasks_runner_no_url( - self, mock_datetime, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test task runner with no URL.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue_instance = Mock() - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - mock_datetime_instance = Mock() - mock_datetime_instance.isoformat.return_value = "2024-01-01T12:00:00Z" - mock_datetime.now.return_value = mock_datetime_instance - - tasks = Tasks(file=None, config=mock_config_instance) - - test_task = Task( - id="task1", - name="Test Task", - url="", # Empty URL to trigger error - enabled=True, - ) - - # Add task to tasks list so it can be found by _runner - tasks._tasks.append(test_task) - - await tasks._runner(test_task) - - # Verify download queue was NOT called - mock_download_queue_instance.add.assert_not_called() - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Item") - @patch("app.library.Tasks.datetime") - async def test_tasks_runner_download_queue_failure( - self, mock_datetime, mock_item, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test task runner when download queue add fails.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler.get_instance.return_value = Mock() - - # Mock download queue to raise exception - mock_download_queue_instance = Mock() - mock_download_queue_instance.add = AsyncMock(side_effect=Exception("Queue error")) - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - mock_datetime_instance = Mock() - mock_datetime_instance.isoformat.return_value = "2024-01-01T12:00:00Z" - mock_datetime.now.return_value = mock_datetime_instance - - mock_item.format.return_value = {"formatted": "item"} - - tasks = Tasks(file=None, config=mock_config_instance) - - test_task = Task( - id="task1", - name="Test Task", - url="https://example.com/video", - enabled=True, - ) - - # Add task to tasks list so it can be found by _runner - tasks._tasks.append(test_task) - - await tasks._runner(test_task) - - # Verify error event was emitted (check last call) - calls = mock_eventbus_instance.emit.call_args_list - assert len(calls) > 0 - # Verify the last call was an error event - last_call = calls[-1] - assert "Task failed" in str(last_call) - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - async def test_tasks_runner_disabled_task(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test that disabled tasks are skipped by the runner.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler.get_instance.return_value = Mock() - - # Mock download queue - mock_download_queue_instance = Mock() - mock_download_queue_instance.add = AsyncMock(return_value={"success": True, "id": "download123"}) - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - tasks = Tasks(file=None, config=mock_config_instance) - - test_task = Task( - id="task1", - name="Test Task", - url="https://example.com/video", - enabled=False, # Task is disabled - ) - - # Add task to tasks list so it can be found by _runner - tasks._tasks.append(test_task) - - await tasks._runner(test_task) - - # Verify download queue was NOT called (task was skipped) - mock_download_queue_instance.add.assert_not_called() - - -class TestHandleTaskInspect: - """Test the HandleTask inspect method with static_only parameter.""" - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Services") - async def test_inspect_with_static_only_true( - self, mock_services, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test inspect with static_only=True returns immediately after can_handle check.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - # Mock Services to simulate successful can_handle - mock_services_instance = Mock() - mock_services_instance.handle_sync = Mock(return_value=True) - # handle_async will be called once for can_handle (since it's now async) - mock_services_instance.handle_async = AsyncMock(return_value=True) - mock_services.get_instance.return_value = mock_services_instance - - # Create a mock handler - mock_handler = Mock() - mock_handler.__name__ = "TestHandler" - mock_handler.can_handle = Mock(return_value=True) - mock_handler.extract = AsyncMock() # Should NOT be called with static_only=True - - tasks = Tasks(file=None, config=mock_config_instance) - handler = tasks.get_handler() - handler._handlers = [mock_handler] - - # Call inspect with static_only=True - result = await handler.inspect( - url="https://example.com/feed", preset="default", handler_name=None, static_only=True - ) - - assert hasattr(result, "items"), "Verify result structure" - assert hasattr(result, "metadata") - assert result.items == [] - assert result.metadata["matched"] is True - assert result.metadata["handler"] == "TestHandler" - - # Verify handle_async was called once for can_handle (now async) - assert mock_services_instance.handle_async.call_count == 1, "Should call handle_async once for can_handle" - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Services") - async def test_inspect_with_static_only_false( - self, mock_services, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test inspect with static_only=False performs full extraction.""" - from app.library.Tasks import TaskResult - - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - # Mock Services - mock_services_instance = Mock() - mock_services_instance.handle_sync = Mock(return_value=True) - - # Mock successful extract result - mock_extract_result = TaskResult( - items=[{"url": "https://example.com/video1", "title": "Video 1"}], - metadata={"raw_count": 1}, - ) - mock_services_instance.handle_async = AsyncMock(return_value=mock_extract_result) - mock_services.get_instance.return_value = mock_services_instance - - # Create a mock handler - mock_handler = Mock() - mock_handler.__name__ = "TestHandler" - mock_handler.can_handle = Mock(return_value=True) - mock_handler.extract = AsyncMock(return_value=mock_extract_result) - - tasks = Tasks(file=None, config=mock_config_instance) - handler = tasks.get_handler() - handler._handlers = [mock_handler] - - # Call inspect with static_only=False (default) - result = await handler.inspect(url="https://example.com/feed", preset="default", handler_name=None) - - assert hasattr(result, "items"), "Verify result structure includes extracted items" - assert hasattr(result, "metadata") - assert len(result.items) > 0 - assert result.metadata["matched"] is True - assert result.metadata["handler"] == "TestHandler" - assert result.metadata["supported"] is True - - # Verify handle_async was called twice: once for can_handle, once for extract - assert mock_services_instance.handle_async.call_count == 2, ( - "Should call handle_async twice: can_handle + extract" - ) - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Services") - async def test_inspect_static_only_with_no_handler_match( - self, mock_services, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test inspect with static_only=True when no handler matches.""" - from app.library.Tasks import TaskFailure - - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - # Mock Services to simulate failed can_handle - mock_services_instance = Mock() - mock_services_instance.handle_sync = Mock(return_value=False) - mock_services.get_instance.return_value = mock_services_instance - - # Create a mock handler that won't match - mock_handler = Mock() - mock_handler.__name__ = "TestHandler" - mock_handler.can_handle = Mock(return_value=False) - - tasks = Tasks(file=None, config=mock_config_instance) - handler = tasks.get_handler() - handler._handlers = [mock_handler] - - # Call inspect with static_only=True - result = await handler.inspect( - url="https://unsupported.com/feed", preset="default", handler_name=None, static_only=True - ) - - # Verify result is a TaskFailure - assert isinstance(result, TaskFailure) - assert result.metadata["matched"] is False - assert result.metadata["handler"] is None diff --git a/ui/app/components/TaskForm.vue b/ui/app/components/TaskForm.vue index da4cfbdf..ba14db0e 100644 --- a/ui/app/components/TaskForm.vue +++ b/ui/app/components/TaskForm.vue @@ -364,18 +364,18 @@ import { useStorage } from '@vueuse/core' import { CronExpressionParser } from 'cron-parser' import TextareaAutocomplete from '~/components/TextareaAutocomplete.vue' import type { AutoCompleteOptions } from '~/types/autocomplete' -import type { exported_task, task_item } from '~/types/tasks' +import type { ExportedTask, Task } from '~/types/tasks' import { useConfirm } from '~/composables/useConfirm' const props = defineProps<{ - reference?: string | null | undefined - task: task_item + reference?: number | null | undefined + task: Task addInProgress?: boolean }>() const emitter = defineEmits<{ (e: 'cancel'): void - (e: 'submit', payload: { reference: string | null | undefined, task: task_item, archive_all?: boolean }): void + (e: 'submit', payload: { reference: number | null | undefined, task: Task, archive_all?: boolean }): void }>() const toast = useNotification() @@ -391,7 +391,7 @@ const archiveAllAfterAdd = ref(false) const CHANNEL_REGEX = /^https?:\/\/(?:www\.)?youtube\.com\/(?:(?:channel\/(?UC[0-9A-Za-z_-]{22}))|(?:c\/(?[A-Za-z0-9_-]+))|(?:user\/(?[A-Za-z0-9_-]+))|(?:@(?[A-Za-z0-9_-]+)))(?\/.*)?\/?$/ const GENERIC_RSS_REGEX = /\.(rss|atom)(\?.*)?$|handler=rss/i -const form = reactive({ ...props.task }) +const form = reactive({ ...props.task }) watch(() => config.ytdlp_options, newOptions => ytDlpOpt.value = newOptions .filter(opt => !opt.ignored) @@ -468,7 +468,7 @@ const importItem = async (): Promise => { } try { - const item = decode(val) as exported_task + const item = decode(val) as ExportedTask if ('task' !== item._type) { toast.error(`Invalid import string. Expected type 'task', got '${item._type}'.`) diff --git a/ui/app/composables/useTasks.ts b/ui/app/composables/useTasks.ts new file mode 100644 index 00000000..4baa692a --- /dev/null +++ b/ui/app/composables/useTasks.ts @@ -0,0 +1,512 @@ +import { ref, readonly } from 'vue' + +import { useNotification } from '~/composables/useNotification' +import { request, parse_list_response, parse_api_response, parse_api_error } from '~/utils' +import type { + Task, + TaskPatch, + TaskInspectRequest, + TaskInspectResponse, + TaskMetadataResponse, +} from '~/types/tasks' +import type { APIResponse, Pagination } from '~/types/responses' + +/** + * List of all tasks in memory. + */ +const tasks = ref>([]) +/** + * Pagination state for tasks list. + */ +const pagination = ref({ + page: 1, + per_page: 50, + total: 0, + total_pages: 0, + has_next: false, + has_prev: false, +}) +/** + * Indicates if a request is in progress. + */ +const isLoading = ref(false) +/** + * Indicates if an add/update operation is in progress. + */ +const addInProgress = ref(false) +/** + * Stores the last error message, if any. + */ +const lastError = ref(null) +/** + * If true, methods will throw errors instead of returning null/false (for testing) + */ +const throwInstead = ref(false) +/** + * Notification composable for showing success/error messages. + */ +const notify = useNotification() + +/** + * Sorts tasks by name (A-Z). + * @param items Array of tasks + * @returns Sorted array of tasks + */ +const sortTasks = (items: Array): Array => { + return [...items].sort((a, b) => a.name.localeCompare(b.name)) +} + +/** + * Safely reads JSON from a Response, returns null on error. + * @param response Fetch Response object + * @returns Parsed JSON or null + */ +const readJson = async (response: Response): Promise => { + try { + const clone = response.clone() + return await clone.json() + } + catch { + return null + } +} + +/** + * Throws an error if the response is not OK, using API error message if available. + * @param response Fetch Response object + * @throws Error with message from API or status code + */ +const ensureSuccess = async (response: Response): Promise => { + if (response.ok) { + return + } + + const payload = await readJson(response) + const message = await parse_api_error(payload) + throw new Error(message) +} + +/** + * Handles errors by updating lastError and showing a notification. + * @param error Error object or unknown + */ +const handleError = (error: unknown): void => { + const message = error instanceof Error ? error.message : 'Unexpected error occurred.' + lastError.value = message + notify.error(message) +} + +/** + * Updates or adds a task in the list, keeping sort order. + * Also increments pagination.total if it's a new task. + * @param item Task to update/add + */ +const updateTasksList = (item: Task): void => { + const isNew = !tasks.value.some(existing => existing.id === item.id) + tasks.value = sortTasks([ + ...tasks.value.filter(existing => existing.id !== item.id), + item, + ]) + if (isNew) { + pagination.value.total++ + } +} + +/** + * Removes a task from the list by ID. + * Also decrements pagination.total. + * @param id Task ID + */ +const removeTask = (id: number) => { + const initialLength = tasks.value.length + tasks.value = tasks.value.filter(item => item.id !== id) + if (tasks.value.length < initialLength) { + pagination.value.total = Math.max(0, pagination.value.total - 1) + } +} + +/** + * Loads tasks from the API with pagination support. + * @param page Page number + * @param perPage Items per page + */ +const loadTasks = async (page: number = 1, perPage: number | undefined = undefined): Promise => { + isLoading.value = true + try { + let url = `/api/tasks/?page=${page}` + if (perPage !== undefined) { + url += `&per_page=${perPage}` + } + const response = await request(url) + await ensureSuccess(response) + + const json = await response.json() + const { items, pagination: paginationData } = await parse_list_response(json) + + tasks.value = sortTasks(items) + pagination.value = paginationData + lastError.value = null + } + catch (error) { + handleError(error) + if (throwInstead.value) throw error + } + finally { + isLoading.value = false + } +} + +/** + * Fetches a single task by ID from the API. + * @param id Task ID + * @returns Task or null on error + */ +const getTask = async (id: number): Promise => { + try { + const response = await request(`/api/tasks/${id}`) + await ensureSuccess(response) + + const json = await response.json() + const task = await parse_api_response(json) + + lastError.value = null + return task + } + catch (error) { + handleError(error) + if (throwInstead.value) throw error + return null + } +} + +/** + * Creates a new task via API. + * @param task Task to create + * @param callback Optional callback with APIResponse result + * @returns Created task or null on error + */ +const createTask = async ( + task: Omit, + callback?: (response: APIResponse) => void, +): Promise => { + addInProgress.value = true + try { + const response = await request('/api/tasks/', { + method: 'POST', + body: JSON.stringify(task), + }) + await ensureSuccess(response) + + const json = await response.json() + const created = await parse_api_response(json) + + updateTasksList(created) + notify.success('Task created.') + lastError.value = null + + if (callback) { + callback({ success: true, error: null, detail: null, data: created }) + } + + return created + } + catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unexpected error occurred.' + handleError(error) + + if (callback) { + callback({ success: false, error: errorMessage, detail: error, data: undefined }) + } + + if (throwInstead.value) throw error + return null + } + finally { + addInProgress.value = false + } +} + +/** + * Updates an existing task via API (PUT - full update). + * @param id Task ID + * @param task Updated task data + * @param callback Optional callback with APIResponse result + * @returns Updated task or null on error + */ +const updateTask = async ( + id: number, + task: Omit, + callback?: (response: APIResponse) => void, +): Promise => { + addInProgress.value = true + try { + // Explicitly remove id, created_at, updated_at fields if present + const { id: _, created_at: __, updated_at: ___, ...taskData } = task as Task + + const response = await request(`/api/tasks/${id}`, { + method: 'PUT', + body: JSON.stringify(taskData), + }) + await ensureSuccess(response) + + const json = await response.json() + const updated = await parse_api_response(json) + + updateTasksList(updated) + notify.success(`Task '${updated.name}' updated.`) + lastError.value = null + + if (callback) { + callback({ success: true, error: null, detail: null, data: updated }) + } + + return updated + } + catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unexpected error occurred.' + handleError(error) + + if (callback) { + callback({ success: false, error: errorMessage, detail: error, data: undefined }) + } + + if (throwInstead.value) throw error + return null + } + finally { + addInProgress.value = false + } +} + +/** + * Partially updates an existing task via API (PATCH). + * @param id Task ID + * @param patch Partial task data to update + * @param callback Optional callback with APIResponse result + * @returns Updated task or null on error + */ +const patchTask = async ( + id: number, + patch: TaskPatch, + callback?: (response: APIResponse) => void, +): Promise => { + addInProgress.value = true + try { + const response = await request(`/api/tasks/${id}`, { + method: 'PATCH', + body: JSON.stringify(patch), + }) + await ensureSuccess(response) + + const json = await response.json() + const updated = await parse_api_response(json) + + updateTasksList(updated) + notify.success(`Task '${updated.name}' updated.`) + lastError.value = null + + if (callback) { + callback({ success: true, error: null, detail: null, data: updated }) + } + + return updated + } + catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unexpected error occurred.' + handleError(error) + + if (callback) { + callback({ success: false, error: errorMessage, detail: error, data: undefined }) + } + + if (throwInstead.value) throw error + return null + } + finally { + addInProgress.value = false + } +} + +/** + * Deletes a task by ID via API. + * @param id Task ID + * @param callback Optional callback with APIResponse result + * @returns true if deleted, false on error + */ +const deleteTask = async ( + id: number, + callback?: (response: APIResponse) => void, +): Promise => { + try { + const response = await request(`/api/tasks/${id}`, { method: 'DELETE' }) + await ensureSuccess(response) + + removeTask(id) + notify.success('Task deleted.') + lastError.value = null + + if (callback) { + callback({ success: true, error: null, detail: null, data: true }) + } + + return true + } + catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unexpected error occurred.' + handleError(error) + + if (callback) { + callback({ success: false, error: errorMessage, detail: error, data: false }) + } + + if (throwInstead.value) throw error + return false + } +} + +/** + * Inspects a task handler to check if it can process the given URL. + * @param request Task inspect request parameters + * @returns Inspect result or null on error + */ +const inspectTaskHandler = async (request: TaskInspectRequest): Promise => { + try { + const response = await fetch('/api/tasks/inspect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + }) + + const json = await response.json() + lastError.value = null + return json as TaskInspectResponse + } + catch (error) { + handleError(error) + if (throwInstead.value) throw error + return null + } +} + +/** + * Marks all items from a task as downloaded in the download archive. + * @param id Task ID + * @returns Success message or null on error + */ +const markTaskItems = async (id: number): Promise => { + try { + const response = await request(`/api/tasks/${id}/mark`, { method: 'POST' }) + await ensureSuccess(response) + + const json = await response.json() + const message = json.message || 'All items marked as downloaded.' + + notify.success(message) + lastError.value = null + return message + } + catch (error) { + handleError(error) + if (throwInstead.value) throw error + return null + } +} + +/** + * Removes all items from a task from the download archive. + * @param id Task ID + * @returns Success message or null on error + */ +const unmarkTaskItems = async (id: number): Promise => { + try { + const response = await request(`/api/tasks/${id}/mark`, { method: 'DELETE' }) + await ensureSuccess(response) + + const json = await response.json() + const message = json.message || 'All items removed from archive.' + + notify.success(message) + lastError.value = null + return message + } + catch (error) { + handleError(error) + if (throwInstead.value) throw error + return null + } +} + +/** + * Generates metadata for a task (tvshow.nfo, info.json, thumbnails). + * @param id Task ID + * @returns Metadata response or null on error + */ +const generateTaskMetadata = async (id: number): Promise => { + try { + const response = await request(`/api/tasks/${id}/metadata`, { method: 'POST' }) + await ensureSuccess(response) + + const json = await response.json() + const metadata = await parse_api_response(json) + + notify.success('Metadata generation completed.') + lastError.value = null + return metadata + } + catch (error) { + handleError(error) + if (throwInstead.value) throw error + return null + } +} + +/** + * Clears the last error message. + */ +const clearError = () => lastError.value = null + +/** + * Resets all state to initial values (for testing only). + */ +const __resetForTesting = () => { + tasks.value = [] + pagination.value = { + page: 1, + per_page: 50, + total: 0, + total_pages: 0, + has_next: false, + has_prev: false, + } + isLoading.value = false + addInProgress.value = false + lastError.value = null + throwInstead.value = false +} + +/** + * useTasks composable + * + * Returns reactive state and CRUD methods for tasks. + * @returns Object with state and API methods + */ +export const useTasks = () => ({ + tasks: readonly(tasks), + pagination: readonly(pagination), + isLoading: readonly(isLoading), + addInProgress: readonly(addInProgress), + lastError: readonly(lastError), + loadTasks, + getTask, + createTask, + updateTask, + patchTask, + deleteTask, + inspectTaskHandler, + markTaskItems, + unmarkTaskItems, + generateTaskMetadata, + clearError, + throwInstead, + __resetForTesting, +}) diff --git a/ui/app/pages/tasks.vue b/ui/app/pages/tasks.vue index e70654e7..9355e522 100644 --- a/ui/app/pages/tasks.vue +++ b/ui/app/pages/tasks.vue @@ -67,8 +67,8 @@
- +
@@ -401,12 +401,6 @@ -
-
- -
-
-
@@ -470,12 +464,15 @@ import { useStorage } from '@vueuse/core' import { CronExpressionParser } from 'cron-parser' import Modal from '~/components/Modal.vue' import { useConfirm } from '~/composables/useConfirm' +import { useTasks } from '~/composables/useTasks' import TaskInspect from '~/components/TaskInspect.vue' -import type { task_item, exported_task, error_response } from '~/types/tasks' +import type { Task, ExportedTask } from '~/types/tasks' import type { WSEP } from '~/types/sockets' import { sleep } from '~/utils' import { useSessionCache } from '~/utils/cache' +type TaskWithUI = Task & { in_progress?: boolean } + const box = useConfirm() const toast = useNotification() const config = useConfigStore() @@ -486,18 +483,19 @@ const sessionCache = useSessionCache() const display_style = useStorage("tasks_display_style", "cards") const isMobile = useMediaQuery({ maxWidth: 1024 }) -const tasks = ref>([]) -const task = ref>({}) -const taskRef = ref('') +// Use the tasks composable +const tasksComposable = useTasks() +const { tasks, isLoading, addInProgress } = tasksComposable + +const task = ref>({}) +const taskRef = ref(null) const toggleForm = ref(false) -const isLoading = ref(true) -const addInProgress = ref(false) -const selectedElms = ref>([]) +const selectedElms = ref>([]) const masterSelectAll = ref(false) const massRun = ref(false) const massDelete = ref(false) const table_container = ref(false) -const inspectTask = ref(null) +const inspectTask = ref(null) const reset_dialog = () => ({ visible: false, @@ -510,8 +508,6 @@ const reset_dialog = () => ({ const dialog_confirm = ref(reset_dialog()) -const remove_keys = ['in_progress'] - const query = ref() const toggleFilter = ref(false) @@ -526,14 +522,17 @@ watch(query, () => { selectedElms.value = [] }) + watch(masterSelectAll, value => { if (!value) { selectedElms.value = [] return } for (const key in filteredTasks.value) { - const element = filteredTasks.value[key] as task_item - selectedElms.value.push(element.id) + const element = filteredTasks.value[key] as Task + if (element.id) { + selectedElms.value.push(element.id) + } } }) @@ -544,13 +543,14 @@ watch(() => socket.isConnected, async () => { socket.on('item_status', statusHandler) }) + const CACHE_KEY = 'tasks:handler_support' const taskHandlerSupport = ref>(sessionCache.get(CACHE_KEY) || {}) watch(taskHandlerSupport, (newValue) => sessionCache.set(CACHE_KEY, newValue), { deep: true }) -const getCacheKey = (task: task_item): string => `${task.id}:${task.url}` +const getCacheKey = (task: Task): string => `${task.id}:${task.url}` -const cleanStaleCache = (currentTasks: Array) => { +const cleanStaleCache = (currentTasks: ReadonlyArray) => { const validKeys = new Set(currentTasks.map(task => getCacheKey(task))) const cacheKeys = Object.keys(taskHandlerSupport.value) @@ -561,7 +561,7 @@ const cleanStaleCache = (currentTasks: Array) => { } } -const recheckHandlerSupport = async (updatedTasks: Array) => { +const recheckHandlerSupport = async (updatedTasks: ReadonlyArray) => { for (const task of updatedTasks) { if (!task.timer && false !== task.handler_enabled) { await checkHandlerSupport(task) @@ -569,7 +569,7 @@ const recheckHandlerSupport = async (updatedTasks: Array) => { } } -const checkHandlerSupport = async (task: task_item): Promise => { +const checkHandlerSupport = async (task: Task): Promise => { const cacheKey = getCacheKey(task) if (undefined !== taskHandlerSupport.value[cacheKey]) { @@ -577,13 +577,11 @@ const checkHandlerSupport = async (task: task_item): Promise => { } try { - const response = await request('/api/tasks/inspect', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ url: task.url, static_only: true }) + const result = await tasksComposable.inspectTaskHandler({ + url: task.url, + static_only: true }) - const data = await response.json() - const supported = true === data?.matched + const supported = result?.matched === true taskHandlerSupport.value[cacheKey] = supported return supported } catch { @@ -592,7 +590,7 @@ const checkHandlerSupport = async (task: task_item): Promise => { } } -const willTaskBeProcessed = (task: task_item): boolean => { +const willTaskBeProcessed = (task: Task): boolean => { if (false === task.enabled) { return false } @@ -604,41 +602,29 @@ const willTaskBeProcessed = (task: task_item): boolean => { return hasTimer || hasHandler } -const filteredTasks = computed(() => { +const filteredTasks = computed(() => { const q = query.value?.toLowerCase(); if (!q) return tasks.value; return tasks.value.filter(task => deepIncludes(task, q, new WeakSet())); -}); +}) as ComputedRef>; const reloadContent = async (fromMounted: boolean = false) => { try { - isLoading.value = true - const response = await request('/api/tasks') + await tasksComposable.loadTasks() - if (fromMounted && !response.ok) { - return + if (tasks.value.length > 0) { + cleanStaleCache(tasks.value) + await recheckHandlerSupport(tasks.value) } - - const data = await response.json() - if (data.length < 1) { - return - } - - tasks.value = data - cleanStaleCache(data) - await recheckHandlerSupport(data) } catch (e) { - if (fromMounted) { - return + if (!fromMounted) { + console.error(e) } - console.error(e) - toast.error('Failed to fetch tasks.') - } finally { - isLoading.value = false } } + const resetForm = (closeForm: boolean = false) => { task.value = { name: '', @@ -652,44 +638,12 @@ const resetForm = (closeForm: boolean = false) => { handler_enabled: true, enabled: true } - taskRef.value = '' - addInProgress.value = false + taskRef.value = null if (closeForm) { toggleForm.value = false } } -const updateTasks = async (items: Array) => { - try { - addInProgress.value = true - - const response = await request('/api/tasks', { - method: 'PUT', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(items.map(item => cleanObject(toRaw(item), remove_keys))), - }) - - const data: Array | error_response = await response.json() - - if ("error" in data) { - toast.error(`Failed to update tasks. ${data.error}`); - return false - } - - tasks.value = data - cleanStaleCache(data) - await recheckHandlerSupport(data) - resetForm(true) - return true - } catch (e: any) { - toast.error(`Failed to update tasks. ${e.message}`); - } finally { - addInProgress.value = false - } -} - const deleteConfirm = () => { if (selectedElms.value.length < 1) { toast.error('No tasks selected.') @@ -710,139 +664,113 @@ const deleteSelected = async () => { massDelete.value = true dialog_confirm.value = reset_dialog() - const itemsToDelete = tasks.value.filter(t => selectedElms.value.includes(t.id)) + const itemsToDelete = tasks.value.filter(t => t.id && selectedElms.value.includes(t.id)) if (itemsToDelete.length < 1) { toast.error('No tasks found to delete.') + massDelete.value = false return } + // Delete tasks sequentially for (const item of itemsToDelete) { - const index = tasks.value.findIndex(t => t?.id === item.id) - if (index > -1) { - tasks.value.splice(index, 1) + if (item.id) { + await tasksComposable.deleteTask(item.id) } } - await nextTick() - const status = await updateTasks(tasks.value) selectedElms.value = [] - if (!status) { - return - } - - toast.success('Tasks deleted.') setTimeout(async () => { await nextTick() massDelete.value = false }, 500) } -const deleteItem = async (item: task_item) => { +const deleteItem = async (item: Task) => { if (true !== (await box.confirm(`Delete '${item.name}' task?`))) { return } - const index = tasks.value.findIndex((t) => t?.id === item.id) - if (index > -1) { - tasks.value.splice(index, 1) - } else { - toast.error('Task not found') + if (!item.id) { + toast.error('Task ID is missing') return } - const status = await updateTasks(tasks.value) - - if (!status) { - return - } - - toast.success('Task deleted.') + await tasksComposable.deleteTask(item.id) } -const toggleEnabled = async (item: task_item) => { +const toggleEnabled = async (item: Task) => { + if (!item.id) { + toast.error('Task ID is missing') + return + } + const newStatus = !item.enabled - const actionText = newStatus ? 'enable' : 'disable' - try { - const response = await request(`/api/tasks/${item.id}`, { - method: 'PATCH', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ enabled: newStatus }) - }) - const data = await response.json() + const updated = await tasksComposable.patchTask(item.id, { enabled: newStatus }) - if (data?.error) { - toast.error(data.error) - return + if (updated) { + // Update local reference + item.enabled = updated.enabled + + // Update handler support cache if needed + if (updated.enabled) { + await checkHandlerSupport(updated) } - - item.enabled = data.enabled - const func = data.enabled ? 'success' : 'warning' - toast[func](`Task '${item.name}' ${data.enabled ? 'enabled' : 'disabled'}.`) - } catch (e: any) { - toast.error(`Failed to ${actionText} task. ${e.message || 'Unknown error.'}`) } } -const toggleHandlerEnabled = async (item: task_item) => { - const newStatus = !item.handler_enabled - const actionText = newStatus ? 'enable' : 'disable' - - try { - const response = await request(`/api/tasks/${item.id}`, { - method: 'PATCH', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ handler_enabled: newStatus }) - }) - const data = await response.json() - - if (data?.error) { - toast.error(data.error) - return - } - - item.handler_enabled = data.handler_enabled - const func = data.handler_enabled ? 'success' : 'warning' - toast[func](`Task handler for '${item.name}' ${data.handler_enabled ? 'enabled' : 'disabled'}.`) - } catch (e: any) { - toast.error(`Failed to ${actionText} task handler. ${e.message || 'Unknown error.'}`) - } -} - -const updateItem = async ({ reference, task, archive_all }: { reference?: string | null | undefined, task: task_item, archive_all?: boolean }) => { - if (reference) { - // -- find the task index. - const index = tasks.value.findIndex((t) => t?.id === reference) - if (index > -1) { - tasks.value[index] = task - } - } else { - tasks.value.push(task) - } - - const status = await updateTasks(tasks.value) - if (!status) { +const toggleHandlerEnabled = async (item: Task) => { + if (!item.id) { + toast.error('Task ID is missing') return } - toast.success('Task updated.') + const newStatus = !item.handler_enabled - if (!reference && true === archive_all) { - const newTask = tasks.value[tasks.value.length - 1] - if (newTask) { - await sleep(1) - await nextTick() - await archiveAll(newTask, true) + const updated = await tasksComposable.patchTask(item.id, { handler_enabled: newStatus }) + + if (updated) { + // Update local reference + item.handler_enabled = updated.handler_enabled + + // Update handler support cache if needed + if (updated.handler_enabled) { + await checkHandlerSupport(updated) } } +} + +const updateItem = async ({ reference, task, archive_all }: { reference?: number | null | undefined, task: Task, archive_all?: boolean }) => { + let createdOrUpdated: Task | null = null + + if (reference) { + // Update existing task + createdOrUpdated = await tasksComposable.updateTask(reference, task) + } else { + // Create new task + createdOrUpdated = await tasksComposable.createTask(task) + } + + if (!createdOrUpdated) { + return + } + + // Check handler support for the new/updated task + await checkHandlerSupport(createdOrUpdated) + + if (!reference && true === archive_all && createdOrUpdated.id) { + await sleep(1) + await nextTick() + await archiveAll(createdOrUpdated, true) + } resetForm(true) } -const editItem = (item: task_item) => { - task.value = item - taskRef.value = item.id +const editItem = (item: Task) => { + task.value = { ...item } + taskRef.value = item.id ?? null toggleForm.value = true } @@ -914,7 +842,7 @@ const runSelected = async () => { }, 500) } -const runNow = async (item: task_item, mass: boolean = false) => { +const runNow = async (item: TaskWithUI, mass: boolean = false) => { if (!mass && true !== (await box.confirm(`Run '${item.name}' now? it will also run at the scheduled time.`))) { return } @@ -923,10 +851,10 @@ const runNow = async (item: task_item, mass: boolean = false) => { item.in_progress = true } - const data = { + const data: Record = { url: item.url, preset: item.preset, - } as task_item + } if (item.folder) { data.folder = item.folder @@ -967,7 +895,7 @@ const statusHandler = async (payload: WSEP['item_status']) => { } } -const exportItem = async (item: task_item) => { +const exportItem = async (item: Task) => { const info = JSON.parse(JSON.stringify(item)) const data = { @@ -979,7 +907,7 @@ const exportItem = async (item: task_item) => { auto_start: info?.auto_start ?? true, handler_enabled: info?.handler_enabled ?? true, enabled: info?.enabled ?? true, - } as exported_task + } as ExportedTask if (info.template) { data.template = info.template @@ -1003,9 +931,13 @@ const get_tags = (name: string): Array => { const remove_tags = (name: string): string => name.replace(/\[(.*?)\]/g, '').trim(); -const archiveAll = async (item: task_item, by_pass: boolean = false) => { - try { +const archiveAll = async (item: TaskWithUI, by_pass: boolean = false) => { + if (!item.id) { + toast.error('Task ID is missing') + return + } + try { if (true !== by_pass) { const { status } = await cDialog({ message: `Mark all '${item.name}' items as downloaded in download archive?` @@ -1017,16 +949,7 @@ const archiveAll = async (item: task_item, by_pass: boolean = false) => { } item.in_progress = true - - const response = await request(`/api/tasks/${item.id}/mark`, { method: 'POST' }) - const data = await response.json() - - if (data?.error) { - toast.error(data.error) - return - } - - toast.success(data.message) + await tasksComposable.markTaskItems(item.id) } catch (e: any) { toast.error(`Failed to archive items. ${e.message || 'Unknown error.'}`) return @@ -1035,7 +958,12 @@ const archiveAll = async (item: task_item, by_pass: boolean = false) => { } } -const unarchiveAll = async (item: task_item) => { +const unarchiveAll = async (item: TaskWithUI) => { + if (!item.id) { + toast.error('Task ID is missing') + return + } + try { const { status } = await cDialog({ message: `Remove all '${item.name}' items from download archive?` @@ -1046,16 +974,7 @@ const unarchiveAll = async (item: task_item) => { } item.in_progress = true - - const response = await request(`/api/tasks/${item.id}/mark`, { method: 'DELETE' }) - const data = await response.json() - - if (data?.error) { - toast.error(data.error) - return - } - - toast.success(data.message) + await tasksComposable.unmarkTaskItems(item.id) } catch (e: any) { toast.error(`Failed to remove items from archive. ${e.message || 'Unknown error.'}`) return @@ -1064,7 +983,12 @@ const unarchiveAll = async (item: task_item) => { } } -const generateMeta = async (item: task_item) => { +const generateMeta = async (item: TaskWithUI) => { + if (!item.id) { + toast.error('Task ID is missing') + return + } + try { const { status } = await cDialog({ rawHTML: ` @@ -1093,15 +1017,7 @@ const generateMeta = async (item: task_item) => { } item.in_progress = true - const response = await request(`/api/tasks/${item.id}/metadata`, { method: 'POST' }) - const data = await response.json() - - if (data?.error) { - toast.error(data.error) - return - } - - toast.success('Metadata generation completed.') + await tasksComposable.generateTaskMetadata(item.id) } catch (e: any) { toast.error(`Failed to generate metadata. ${e.message || 'Unknown error.'}`) return diff --git a/ui/app/types/tasks.ts b/ui/app/types/tasks.ts index ccd6b641..183c4dce 100644 --- a/ui/app/types/tasks.ts +++ b/ui/app/types/tasks.ts @@ -1,20 +1,116 @@ -type task_item = { - id: string, - name: string, - url: string, - preset?: string, - folder?: string, - template?: string, - cli?: string, - timer?: string, - in_progress?: boolean, - auto_start?: boolean, - handler_enabled?: boolean, - enabled?: boolean, +import type { Paginated } from '~/types/responses' + +/** + * Main Task interface matching backend Task schema. + */ +export interface Task { + id?: number + name: string + url: string + folder?: string + preset?: string + timer?: string + template?: string + cli?: string + auto_start?: boolean + handler_enabled?: boolean + enabled?: boolean + created_at?: string + updated_at?: string } -type exported_task = task_item & { _type: string, _version: string } +/** + * Partial Task interface for PATCH operations. + */ +export type TaskPatch = Omit & { + name?: string + url?: string +} -type error_response = { error: string } +/** + * Paginated list of tasks response. + */ +export type TaskList = Paginated -export type { task_item, exported_task, error_response }; +/** + * Task handler inspect request. + */ +export interface TaskInspectRequest { + url: string + preset?: string + handler?: string + static_only?: boolean +} + +/** + * Task handler inspect response - success. + */ +export interface TaskInspectSuccess { + matched: true + handler: string + message: string +} + +/** + * Task handler inspect response - failure. + */ +export interface TaskInspectFailure { + matched: false + message: string + error: string +} + +/** + * Task handler inspect response (union type). + */ +export type TaskInspectResponse = TaskInspectSuccess | TaskInspectFailure + +/** + * Task metadata response. + */ +export interface TaskMetadataResponse { + id: string + id_type: string | null + title: string | null + description: string + uploader: string + tags: Array + year: number | null + thumbnails: Record + json_file?: string + nfo_file?: string +} + +/** + * Exported task for import/export functionality. + */ +export interface ExportedTask extends Omit { + _type: string + _version: string +} + +/** + * Generic error response. + */ +export interface ErrorResponse { + error: string + detail?: unknown +} + +/** + * Legacy alias for backward compatibility (deprecated). + * @deprecated Use Task instead + */ +export type task_item = Task + +/** + * Legacy alias for backward compatibility (deprecated). + * @deprecated Use ExportedTask instead + */ +export type exported_task = ExportedTask + +/** + * Legacy alias for backward compatibility (deprecated). + * @deprecated Use ErrorResponse instead + */ +export type error_response = ErrorResponse diff --git a/ui/app/utils/index.ts b/ui/app/utils/index.ts index 3e0c393d..49210ad3 100644 --- a/ui/app/utils/index.ts +++ b/ui/app/utils/index.ts @@ -1,5 +1,5 @@ import { useStorage } from '@vueuse/core' -import type { convert_args_response,Paginated } from '~/types/responses' +import type { convert_args_response, Paginated } from '~/types/responses' import type { StoreItem } from '~/types/store' const AG_SEPARATOR = '.' @@ -858,9 +858,38 @@ const parse_api_error = async (json: unknown): Promise => { const payload = json as { error?: string; message?: string; - detail?: string[]; + detail?: string | Array<{ loc: string[]; msg: string; type: string }>; } - return String(payload.error ?? payload.message ?? payload.detail?.[0] ?? 'Unknown error occurred') + + let extra_detail = '' + + if (Array.isArray(payload.detail)) { + const errors = payload.detail.map((err: any) => { + if ('object' === typeof err && err.loc && err.msg) { + const field = Array.isArray(err.loc) ? err.loc[err.loc.length - 1] : 'unknown' + return `${field}: ${err.msg}` + } + return String(err) + }) + extra_detail = errors.join(', ') + } + + if (payload.error) { + return String(payload.error+(extra_detail ? ` - ${extra_detail}` : '')) + } + if (payload.message) { + return String(payload.message+(extra_detail ? ` - ${extra_detail}` : '')) + } + + if (extra_detail) { + return extra_detail + } + + if ('string' === typeof payload.detail) { + return payload.detail + } + + return 'Unknown error occurred' } export { diff --git a/ui/tests/composables/useTasks.test.ts b/ui/tests/composables/useTasks.test.ts new file mode 100644 index 00000000..c5cabe45 --- /dev/null +++ b/ui/tests/composables/useTasks.test.ts @@ -0,0 +1,860 @@ +import * as utils from '~/utils/index' +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { useTasks } from '~/composables/useTasks' +import type { Task, TaskInspectResponse, TaskMetadataResponse, Pagination } from '~/types/tasks' + +vi.mock('~/composables/useNotification', () => { + const success = vi.fn() + const error = vi.fn() + return { + useNotification: () => ({ success, error }), + default: () => ({ success, error }), + } +}) + +// Sample data +const mockTask: Task = { + id: 1, + name: 'Test Task', + url: 'https://www.youtube.com/channel/UCtest123', + folder: 'youtube/Test', + preset: 'default', + timer: '0 12 * * *', + template: '%(title)s.%(ext)s', + cli: '--format best', + auto_start: true, + handler_enabled: true, + enabled: true, + created_at: '2024-01-01T00:00:00Z', + updated_at: '2024-01-01T00:00:00Z', +} + +const mockPagination: Pagination = { + page: 1, + per_page: 50, + total: 1, + total_pages: 1, + has_next: false, + has_prev: false, +} + +// Helper to create a mock Response object +function createMockResponse({ ok, status, jsonData }: { ok: boolean; status: number; jsonData: any }) { + return { + ok, + status, + headers: new Headers(), + redirected: false, + statusText: '', + type: 'basic', + url: '', + body: null, + bodyUsed: false, + clone() { + return this + }, + async json() { + return jsonData + }, + text: async () => JSON.stringify(jsonData), + arrayBuffer: async () => new ArrayBuffer(0), + blob: async () => new Blob(), + formData: async () => new FormData(), + } as Response +} + +describe('useTasks', () => { + beforeEach(() => { + vi.clearAllMocks() + // Reset composable state between tests + const tasks = useTasks() + tasks.__resetForTesting() + }) + + describe('loadTasks', () => { + it('loads tasks with pagination successfully', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { + items: [mockTask], + pagination: mockPagination, + }, + }), + ) + + const tasks = useTasks() + await tasks.loadTasks() + + expect(tasks.tasks.value).toHaveLength(1) + expect(tasks.tasks.value[0]).toEqual(mockTask) + expect(tasks.pagination.value).toEqual(mockPagination) + expect(tasks.lastError.value).toBeNull() + }) + + it('sorts tasks by name A-Z', async () => { + const items = [ + { ...mockTask, id: 1, name: 'Zebra' }, + { ...mockTask, id: 2, name: 'Alpha' }, + { ...mockTask, id: 3, name: 'Beta' }, + ] + + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { + items, + pagination: mockPagination, + }, + }), + ) + + const tasks = useTasks() + await tasks.loadTasks() + + const sorted = tasks.tasks.value + expect(sorted[0].name).toBe('Alpha') + expect(sorted[1].name).toBe('Beta') + expect(sorted[2].name).toBe('Zebra') + }) + + it('handles empty tasks list', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { + items: [], + pagination: { ...mockPagination, total: 0 }, + }, + }), + ) + + const tasks = useTasks() + await tasks.loadTasks() + + expect(tasks.tasks.value).toEqual([]) + expect(tasks.lastError.value).toBeNull() + }) + + it('handles errors during load', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 500, + jsonData: { error: 'Server error' }, + }), + ) + + const tasks = useTasks() + await tasks.loadTasks() + + expect(tasks.lastError.value).toBeTruthy() + }) + + it('respects page and per_page parameters', async () => { + const requestSpy = vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { + items: [mockTask], + pagination: { ...mockPagination, page: 2 }, + }, + }), + ) + + const tasks = useTasks() + await tasks.loadTasks(2, 25) + + expect(requestSpy).toHaveBeenCalledWith('/api/tasks/?page=2&per_page=25') + }) + }) + + describe('getTask', () => { + it('fetches a single task successfully', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: mockTask, + }), + ) + + const tasks = useTasks() + const result = await tasks.getTask(1) + + expect(result).toEqual(mockTask) + expect(tasks.lastError.value).toBeNull() + }) + + it('handles 404 not found', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 404, + jsonData: { error: 'Task not found' }, + }), + ) + + const tasks = useTasks() + const result = await tasks.getTask(999) + + expect(result).toBeNull() + expect(tasks.lastError.value).toBeTruthy() + }) + }) + + describe('createTask', () => { + it('creates a task successfully', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: mockTask, + }), + ) + + const tasks = useTasks() + const initialTotal = tasks.pagination.value.total + const newTask = { ...mockTask } + delete (newTask as any).id + + const result = await tasks.createTask(newTask) + + expect(result).toEqual(mockTask) + expect(tasks.tasks.value).toContainEqual(mockTask) + expect(tasks.pagination.value.total).toBe(initialTotal + 1) + }) + + it('calls callback on success', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: mockTask, + }), + ) + + const callback = vi.fn() + const tasks = useTasks() + const newTask = { ...mockTask } + delete (newTask as any).id + + await tasks.createTask(newTask, callback) + + expect(callback).toHaveBeenCalledWith({ + success: true, + error: null, + detail: null, + data: mockTask, + }) + }) + + it('calls callback on error', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 400, + jsonData: { detail: [{ loc: ['name'], msg: 'Field required', type: 'value_error' }] }, + }), + ) + + const callback = vi.fn() + const tasks = useTasks() + const newTask = { ...mockTask } + delete (newTask as any).id + + await tasks.createTask(newTask, callback) + + expect(callback).toHaveBeenCalledWith( + expect.objectContaining({ + success: false, + error: expect.any(String), + }), + ) + }) + }) + + describe('updateTask', () => { + it('updates a task successfully', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { ...mockTask, name: 'Updated Name' }, + }), + ) + + const tasks = useTasks() + const updated = { ...mockTask, name: 'Updated Name' } + + const result = await tasks.updateTask(1, updated) + + expect(result?.name).toBe('Updated Name') + }) + + it('calls callback on success', async () => { + const updatedTask = { ...mockTask, name: 'Updated' } + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: updatedTask, + }), + ) + + const callback = vi.fn() + const tasks = useTasks() + + await tasks.updateTask(1, updatedTask, callback) + + expect(callback).toHaveBeenCalledWith({ + success: true, + error: null, + detail: null, + data: updatedTask, + }) + }) + + it('removes id field from task before sending', async () => { + const requestSpy = vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: mockTask, + }), + ) + + const tasks = useTasks() + const taskWithId = { ...mockTask } + await tasks.updateTask(1, taskWithId) + + const requestBody = JSON.parse((requestSpy.mock.calls[0][1] as any).body) + // The id field should be removed before sending + expect('id' in requestBody).toBe(false) + }) + }) + + describe('patchTask', () => { + it('patches a task successfully', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { ...mockTask, enabled: false }, + }), + ) + + const tasks = useTasks() + const result = await tasks.patchTask(1, { enabled: false }) + + expect(result?.enabled).toBe(false) + }) + + it('calls callback on patch success', async () => { + const patchedTask = { ...mockTask, enabled: false } + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: patchedTask, + }), + ) + + const callback = vi.fn() + const tasks = useTasks() + + await tasks.patchTask(1, { enabled: false }, callback) + + expect(callback).toHaveBeenCalledWith({ + success: true, + error: null, + detail: null, + data: patchedTask, + }) + }) + + it('handles validation errors with callback', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 400, + jsonData: { detail: [{ loc: ['timer'], msg: 'Invalid CRON expression', type: 'value_error' }] }, + }), + ) + + const callback = vi.fn() + const tasks = useTasks() + + await tasks.patchTask(1, { timer: 'invalid' }, callback) + + expect(callback).toHaveBeenCalledWith( + expect.objectContaining({ + success: false, + error: expect.any(String), + }), + ) + }) + }) + + describe('deleteTask', () => { + it('deletes a task successfully', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: mockTask, + }), + ) + + const tasks = useTasks() + const initialTotal = tasks.pagination.value.total + + const result = await tasks.deleteTask(mockTask.id!) + + expect(result).toBe(true) + expect(tasks.pagination.value.total).toBe(Math.max(0, initialTotal - 1)) + }) + + it('calls callback on delete success', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: mockTask, + }), + ) + + const callback = vi.fn() + const tasks = useTasks() + + await tasks.deleteTask(1, callback) + + expect(callback).toHaveBeenCalledWith({ + success: true, + error: null, + detail: null, + data: true, + }) + }) + + it('calls callback on delete error', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 404, + jsonData: { error: 'Task not found' }, + }), + ) + + const callback = vi.fn() + const tasks = useTasks() + + await tasks.deleteTask(999, callback) + + expect(callback).toHaveBeenCalledWith( + expect.objectContaining({ + success: false, + error: expect.any(String), + data: false, + }), + ) + }) + }) + + describe('inspectTaskHandler', () => { + it('inspects task handler successfully', async () => { + const inspectResponse: TaskInspectResponse = { + matched: true, + handler: 'YoutubeHandler', + supported: true, + items: [ + { + url: 'https://www.youtube.com/watch?v=test123', + title: 'Test Video', + archive_id: 'youtube test123', + metadata: { published: '2024-01-01T00:00:00Z' }, + }, + ], + metadata: { + feed_url: 'https://www.youtube.com/feeds/videos.xml?channel_id=UCtest123', + entry_count: 1, + }, + } + + global.fetch = vi.fn().mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: inspectResponse, + }), + ) + + const tasks = useTasks() + const result = await tasks.inspectTaskHandler({ + url: 'https://www.youtube.com/channel/UCtest123', + }) + + expect(result).toEqual(inspectResponse) + expect(result?.matched).toBe(true) + expect(result?.handler).toBe('YoutubeHandler') + }) + + it('handles handler not supported', async () => { + const inspectResponse: TaskInspectResponse = { + matched: false, + handler: null, + supported: false, + items: [], + metadata: null, + } + + global.fetch = vi.fn().mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: inspectResponse, + }), + ) + + const tasks = useTasks() + const result = await tasks.inspectTaskHandler({ + url: 'https://unsupported.com', + }) + + expect(result?.supported).toBe(false) + expect(result?.matched).toBe(false) + }) + + it('handles inspect errors', async () => { + global.fetch = vi.fn().mockRejectedValueOnce(new Error('Network error')) + + const tasks = useTasks() + const result = await tasks.inspectTaskHandler({ + url: 'invalid', + }) + + expect(result).toBeNull() + expect(tasks.lastError.value).toBeTruthy() + }) + }) + + describe('markTaskItems', () => { + it('marks all items as downloaded successfully', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { message: 'Marked 15 items' }, + }), + ) + + const tasks = useTasks() + const result = await tasks.markTaskItems(1) + + expect(result).toBe('Marked 15 items') + expect(tasks.lastError.value).toBeNull() + }) + + it('handles mark errors', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 404, + jsonData: { error: 'Task not found' }, + }), + ) + + const tasks = useTasks() + const result = await tasks.markTaskItems(999) + + expect(result).toBeNull() + expect(tasks.lastError.value).toBeTruthy() + }) + }) + + describe('unmarkTaskItems', () => { + it('unmarks items successfully', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { message: 'Unmarked 10 items' }, + }), + ) + + const tasks = useTasks() + const result = await tasks.unmarkTaskItems(1) + + expect(result).toBe('Unmarked 10 items') + expect(tasks.lastError.value).toBeNull() + }) + + it('handles unmark errors', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 404, + jsonData: { error: 'Task not found' }, + }), + ) + + const tasks = useTasks() + const result = await tasks.unmarkTaskItems(999) + + expect(result).toBeNull() + expect(tasks.lastError.value).toBeTruthy() + }) + }) + + describe('generateTaskMetadata', () => { + it('generates metadata successfully', async () => { + const metadataResponse: TaskMetadataResponse = { + status: 'success', + generated: ['tvshow.nfo', 'info.json', 'poster.jpg'], + } + + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: metadataResponse, + }), + ) + + const tasks = useTasks() + const result = await tasks.generateTaskMetadata(1) + + expect(result).toEqual(metadataResponse) + expect(result?.status).toBe('success') + expect(tasks.lastError.value).toBeNull() + }) + + it('handles metadata generation errors', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 500, + jsonData: { error: 'Failed to generate metadata' }, + }), + ) + + const tasks = useTasks() + const result = await tasks.generateTaskMetadata(1) + + expect(result).toBeNull() + expect(tasks.lastError.value).toBeTruthy() + }) + }) + + describe('error handling', () => { + it('throws when throwInstead is true', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 500, + jsonData: { error: 'Server error' }, + }), + ) + + const tasks = useTasks() + tasks.throwInstead.value = true + + await expect(tasks.loadTasks()).rejects.toThrow() + }) + + it('clears error on clearError call', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 500, + jsonData: { error: 'Server error' }, + }), + ) + + const tasks = useTasks() + tasks.throwInstead.value = false + + await tasks.loadTasks() + + // Error should be set after failed load + expect(tasks.lastError.value).toBeTruthy() + + tasks.clearError() + + expect(tasks.lastError.value).toBeNull() + }) + + it('parses API validation errors correctly', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 400, + jsonData: { + detail: [ + { loc: ['name'], msg: 'Field required', type: 'value_error.missing' }, + { loc: ['url'], msg: 'Invalid URL format', type: 'value_error.url' }, + ], + }, + }), + ) + + const tasks = useTasks() + // Clear any previous error + tasks.clearError() + + const newTask = { ...mockTask, name: '', url: '' } + delete (newTask as any).id + + const result = await tasks.createTask(newTask) + + expect(result).toBeNull() + expect(tasks.lastError.value).toBeTruthy() + expect(tasks.lastError.value).toContain('name: Field required') + expect(tasks.lastError.value).toContain('url: Invalid URL format') + }) + }) + + describe('addInProgress state', () => { + it('sets addInProgress during create operation', async () => { + let inProgressDuringCall = false + + vi.spyOn(utils, 'request').mockImplementation(async (_url, _options) => { + // Get fresh instance to check current state + const currentTasks = useTasks() + inProgressDuringCall = currentTasks.addInProgress.value + return createMockResponse({ + ok: true, + status: 200, + jsonData: mockTask, + }) + }) + + const tasks = useTasks() + const newTask = { ...mockTask } + delete (newTask as any).id + + await tasks.createTask(newTask) + + expect(inProgressDuringCall).toBe(true) + expect(tasks.addInProgress.value).toBe(false) + }) + + it('resets addInProgress on error', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: false, + status: 400, + jsonData: { error: 'Bad request' }, + }), + ) + + const tasks = useTasks() + const newTask = { ...mockTask } + delete (newTask as any).id + + await tasks.createTask(newTask) + + expect(tasks.addInProgress.value).toBe(false) + }) + }) + + describe('isLoading state', () => { + it('sets isLoading during loadTasks operation', async () => { + let loadingDuringCall = false + + vi.spyOn(utils, 'request').mockImplementation(async (_url, _options) => { + // Get fresh instance to check current state + const currentTasks = useTasks() + loadingDuringCall = currentTasks.isLoading.value + return createMockResponse({ + ok: true, + status: 200, + jsonData: { + items: [mockTask], + pagination: mockPagination, + }, + }) + }) + + const tasks = useTasks() + await tasks.loadTasks() + + expect(loadingDuringCall).toBe(true) + expect(tasks.isLoading.value).toBe(false) + }) + }) + + describe('task list updates', () => { + it('updates existing task in list', async () => { + // Load initial task + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { + items: [mockTask], + pagination: mockPagination, + }, + }), + ) + + const tasks = useTasks() + await tasks.loadTasks() + + expect(tasks.tasks.value).toHaveLength(1) + expect(tasks.tasks.value[0].name).toBe('Test Task') + + // Update task + const updatedTask = { ...mockTask, name: 'Updated Name' } + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: updatedTask, + }), + ) + + await tasks.updateTask(1, updatedTask) + + expect(tasks.tasks.value[0].name).toBe('Updated Name') + }) + + it('removes deleted task from list', async () => { + // Load initial task + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: { + items: [mockTask], + pagination: { ...mockPagination, total: 1 }, + }, + }), + ) + + const tasks = useTasks() + await tasks.loadTasks() + + expect(tasks.tasks.value).toHaveLength(1) + expect(tasks.pagination.value.total).toBe(1) + + // Delete task + vi.spyOn(utils, 'request').mockResolvedValueOnce( + createMockResponse({ + ok: true, + status: 200, + jsonData: mockTask, + }), + ) + + await tasks.deleteTask(1) + + expect(tasks.tasks.value).toHaveLength(0) + expect(tasks.pagination.value.total).toBe(0) + }) + }) +})