diff --git a/.vscode/settings.json b/.vscode/settings.json index 2e166d45..c80eb119 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -41,6 +41,7 @@ "copyts", "creationflags", "cronsim", + "crosshairs", "currsize", "dailymotion", "datas", @@ -81,6 +82,8 @@ "hookspath", "httpx", "imagetools", + "jmespath", + "jsonschema", "kibibytes", "lastgroup", "levelno", @@ -112,6 +115,7 @@ "noninteractive", "noprogress", "onefile", + "parsel", "pathex", "pickleable", "platformdirs", @@ -195,9 +199,39 @@ "json.schemas": [ { "fileMatch": [ - "**/var/config/tasks/*.json" + "**/tasks/*.json" ], - "url": "./app/library/task_handlers/task_definition.schema.json" + "url": "./app/schema/task_definition.json" + }, + { + "fileMatch": [ + "**/config/conditions.json" + ], + "url": "./app/schema/conditions.json" + }, + { + "fileMatch": [ + "**/config/dl_fields.json" + ], + "url": "./app/schema/dl_fields.json" + }, + { + "fileMatch": [ + "**/config/notifications.json" + ], + "url": "./app/schema/notifications.json" + }, + { + "fileMatch": [ + "**/config/presets.json" + ], + "url": "./app/schema/presets.json" + }, + { + "fileMatch": [ + "**/config/tasks.json" + ], + "url": "./app/schema/tasks.json" } ] } diff --git a/FAQ.md b/FAQ.md index e8503381..1473b302 100644 --- a/FAQ.md +++ b/FAQ.md @@ -262,7 +262,7 @@ Definitions are reloaded automatically when files change, so you can tweak them `var/config/tasks/01-*.json` for sample files. > [!NOTE] -> A machine-readable schema is available at `app/library/task_handlers/task_definition.schema.json` if you want to validate your JSON with editors or CI tools. +> A machine-readable schema is available at `app/schema/task_definition.json` if you want to validate your JSON with editors or CI tools. # How to generate POT tokens? diff --git a/app/library/Presets.py b/app/library/Presets.py index c6a83011..ff8eadf8 100644 --- a/app/library/Presets.py +++ b/app/library/Presets.py @@ -267,7 +267,7 @@ class Presets(metaclass=Singleton): if not isinstance(item, dict): if not isinstance(item, Preset): msg = f"Unexpected '{type(item).__name__}' type was given." - raise ValueError(msg) # noqa: TRY004 + raise ValueError(msg) item = item.serialize() diff --git a/app/library/TaskDefinitions.py b/app/library/TaskDefinitions.py new file mode 100644 index 00000000..189aa347 --- /dev/null +++ b/app/library/TaskDefinitions.py @@ -0,0 +1,435 @@ +import json +import logging +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from aiohttp import web +from jsonschema import Draft7Validator, SchemaError, ValidationError + +from .config import Config +from .encoder import Encoder +from .Services import Services +from .Singleton import Singleton + +LOG: logging.Logger = logging.getLogger("task_definitions") + + +@dataclass(slots=True) +class TaskDefinitionRecord: + identifier: str + """UUID identifier of the task definition.""" + filename: str + """Filename of the task definition JSON file.""" + name: str + """Human-readable name of the task definition.""" + priority: int + """Priority of the task definition.""" + path: Path + """Path to the task definition JSON file.""" + data: dict[str, Any] + """The task definition data.""" + updated_at: float + """Last modified timestamp of the task definition file.""" + + def serialize(self, *, include_definition: bool = False) -> dict[str, Any]: + """ + Serialize the task definition record to a dictionary. + + Args: + include_definition (bool): Whether to include the full task definition data. + + Returns: + dict[str, Any]: The serialized task definition record. + + """ + payload: dict[str, Any] = { + "id": self.identifier, + "name": self.name, + "priority": self.priority, + "updated_at": self.updated_at, + } + + if include_definition: + payload["definition"] = self.data + + return payload + + def json(self, *, include_definition: bool = False) -> str: + """ + Serialize the task definition record to a JSON string. + + Args: + include_definition (bool): Whether to include the full task definition data. + + Returns: + str: The JSON string representation of the task definition record. + + """ + return Encoder().encode(self.serialize(include_definition=include_definition)) + + +class TaskDefinitions(metaclass=Singleton): + def __init__( + self, + directory: str | Path | None = None, + config: Config | None = None, + validator: Draft7Validator | None = None, + ): + self._config: Config = config or Config.get_instance() + "Instance of Config to use." + self._directory: Path = Path(directory) if directory else Path(self._config.config_path) / "tasks" + "Directory where task definition files are stored." + self._validator: Draft7Validator | None = validator + "JSON schema validator instance." + self._items: dict[str, TaskDefinitionRecord] = {} + "Mapping of task definition ID to TaskDefinitionRecord." + self._schema: Path = Path(self._config.app_path) / "schema" / "task_definition.json" + "Path to the JSON schema file for task definitions." + + try: + self._directory.mkdir(parents=True, exist_ok=True) + except Exception as e: + LOG.error(f"Failed to create tasks directory '{self._directory}': {e}") + + self.load() + + @staticmethod + def get_instance( + directory: str | Path | None = None, + config: Config | None = None, + validator: Draft7Validator | None = None, + ) -> "TaskDefinitions": + """ + Get the singleton instance of TaskDefinitions. + + Args: + directory (str | Path | None): Optional directory to store task definitions. + config (Config | None): Optional Config instance to use. + validator (Draft7Validator | None): Optional JSON schema validator to use. + + Returns: + TaskDefinitions: The singleton instance of TaskDefinitions. + + """ + return TaskDefinitions(directory=directory, config=config, validator=validator) + + def attach(self, _: web.Application) -> None: + """ + Attach the TaskDefinitions service to the application. + + Args: + _ (web.Application): The aiohttp web application instance. + + """ + Services.get_instance().add("task_definitions", self) + + async def on_shutdown(self, _: web.Application) -> None: + """ + Handle application shutdown event. + + Args: + _ (web.Application): The aiohttp web application instance. + + """ + return + + def _get_validator(self) -> Draft7Validator: + """ + Get or create the JSON schema validator for task definitions. + + Returns: + Draft7Validator: The JSON schema validator instance. + + """ + if self._validator: + return self._validator + + try: + contents: str = self._schema.read_text(encoding="utf-8") + schema = json.loads(contents) + except Exception as e: + LOG.error(f"Failed to read task definition schema '{self._schema}': {e}") + raise + + try: + self._validator = Draft7Validator(schema) + except SchemaError as e: + LOG.error(f"Invalid task definition schema '{self._schema}': {e}") + raise + + return self._validator + + def validate(self, definition: dict[str, Any]) -> None: + """ + Validate a task definition against the JSON schema. + + Args: + definition (dict[str, Any]): The task definition to validate. + + Raises: + ValueError: If the task definition is invalid. + + """ + try: + self._get_validator().validate(definition) + except ValidationError as e: + path: str = " ".join(str(part) for part in e.path) + error_path: str = f" ({path})" if path else "" + message: str = f"Task definition validation failed{error_path}: {e.message}" + raise ValueError(message) from e + + def load(self) -> "TaskDefinitions": + """ + Load all task definitions from the directory. + + Returns: + TaskDefinitions: The current instance for chaining. + + """ + self._items.clear() + + if not self._directory.exists(): + return self + + for file_path in sorted(self._directory.glob("*.json")): + stem: str = file_path.stem + + try: + identifier_uuid = uuid.UUID(stem) + except ValueError: + LOG.warning(f"Skipping task definition with invalid UUID filename '{file_path.name}'.") + continue + + if 4 != identifier_uuid.version: + LOG.warning(f"Skipping task definition '{file_path.name}', Name is not UUIDv4.") + continue + + try: + contents: str = file_path.read_text(encoding="utf-8") + except Exception as e: + LOG.error(f"Failed to load task definition '{file_path}': {e!s}") + continue + + try: + parsed = json.loads(contents) + except Exception as e: + LOG.error(f"Failed to parse task definition '{file_path}': {e!s}") + continue + + if not isinstance(parsed, dict): + LOG.error(f"Invalid task definition file '{file_path}': must be a JSON object.") + continue + + data: dict[str, Any] = parsed + + identifier: str = str(identifier_uuid) + + name_value: str = str(data.get("name") or identifier) + priority: int = self._normalize_priority(data.get("priority", 0)) + data["priority"] = priority + + record = TaskDefinitionRecord( + identifier=identifier, + filename=file_path.name, + name=name_value, + priority=priority, + path=file_path, + data=data, + updated_at=file_path.stat().st_mtime, + ) + self._items[record.identifier] = record + + return self + + def list(self) -> list[TaskDefinitionRecord]: + """ + List all task definitions, sorted by priority and name. + + Returns: + list[TaskDefinitionRecord]: List of task definitions sorted by priority and name. + + """ + return sorted( + self._items.values(), + key=lambda record: (record.priority, record.name.lower()), + ) + + def get(self, identifier: str) -> TaskDefinitionRecord | None: + """ + Get a task definition by its identifier. + + Args: + identifier (str): The UUID identifier of the task definition. + + Returns: + TaskDefinitionRecord | None: The task definition record, or None if not found. + + """ + return self._items.get(identifier) + + def _path_for(self, identifier: str) -> Path: + """ + Get the file path for a given task definition identifier. + + Args: + identifier (str): The UUID identifier of the task definition. + + Returns: + Path: The file path for the task definition JSON file. + + """ + return self._directory / f"{identifier}.json" + + def _write_file(self, path: Path, payload: dict[str, Any]) -> None: + """ + Write a task definition to a file. + + Args: + path (Path): The file path to write to. + payload (dict[str, Any]): The task definition data to write. + + Raises: + Exception: If writing to the file fails. + + """ + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + try: + path.chmod(0o600) + except Exception: + pass + + def _normalize_priority(self, value: Any) -> int: + """ + Normalize the priority value to an integer. + + Args: + value (Any): The priority value to normalize. + + Returns: + int: The normalized priority value. + + """ + try: + priority = int(value) + except Exception: + priority = 0 + + return priority + + def _refresh_generic_handler(self) -> None: + """ + Refresh the generic task handler definitions. + """ + try: + from .task_handlers.generic import GenericTaskHandler + + GenericTaskHandler.refresh_definitions(force=True) + except Exception as e: + LOG.error(f"Failed to refresh generic task handler: {e}") + + def create(self, definition: dict[str, Any]) -> TaskDefinitionRecord: + """ + Create a new task definition. + + Args: + definition (dict[str, Any]): The task definition data. + + Returns: + TaskDefinitionRecord: The created task definition record. + + """ + self.validate(definition) + + identifier: str = str(uuid.uuid4()) + path: Path = self._path_for(identifier) + + while path.exists(): + identifier = str(uuid.uuid4()) + path = self._path_for(identifier) + + priority: int = self._normalize_priority(definition.get("priority", 0)) + definition["priority"] = priority + + self._write_file(path, definition) + + record = TaskDefinitionRecord( + identifier=identifier, + filename=path.name, + name=str(definition.get("name", identifier)), + priority=priority, + path=path, + data=definition, + updated_at=path.stat().st_mtime, + ) + + self._items[identifier] = record + self._refresh_generic_handler() + return record + + def update(self, identifier: str, definition: dict[str, Any]) -> TaskDefinitionRecord: + """ + Update an existing task definition. + + Args: + identifier (str): The UUID identifier of the task definition to update. + definition (dict[str, Any]): The updated task definition data. + + Returns: + TaskDefinitionRecord: The updated task definition record. + + Raises: + ValueError: If the task definition does not exist or is invalid. + + """ + record: TaskDefinitionRecord | None = self.get(identifier) + if not record: + message: str = f"Task definition '{identifier}' does not exist." + raise ValueError(message) + + self.validate(definition) + priority: int = self._normalize_priority(definition.get("priority", record.priority)) + definition["priority"] = priority + self._write_file(record.path, definition) + + updated_record = TaskDefinitionRecord( + identifier=identifier, + filename=record.filename, + name=str(definition.get("name", identifier)), + priority=priority, + path=record.path, + data=definition, + updated_at=record.path.stat().st_mtime, + ) + + self._items[identifier] = updated_record + self._refresh_generic_handler() + return updated_record + + def delete(self, identifier: str) -> None: + """ + Delete a task definition. + + Args: + identifier (str): The UUID identifier of the task definition to delete. + + Raises: + ValueError: If the task definition does not exist. + + """ + record: TaskDefinitionRecord | None = self.get(identifier) + if not record: + message: str = f"Task definition '{identifier}' does not exist." + raise ValueError(message) + + try: + record.path.unlink(missing_ok=False) + except FileNotFoundError: + LOG.warning(f"Task definition file '{record.path}' already removed.") + except Exception as exc: + LOG.error(f"Failed to delete task definition '{identifier}': {exc}") + raise + + self._items.pop(identifier, None) + self._refresh_generic_handler() diff --git a/app/library/Tasks.py b/app/library/Tasks.py index b3318520..38ef7281 100644 --- a/app/library/Tasks.py +++ b/app/library/Tasks.py @@ -417,7 +417,7 @@ class Tasks(metaclass=Singleton): if not isinstance(task, dict): if not isinstance(task, Task): msg = "Invalid task type." - raise ValueError(msg) # noqa: TRY004 + raise ValueError(msg) task = task.serialize() diff --git a/app/library/conditions.py b/app/library/conditions.py index adb813ed..cdc9e954 100644 --- a/app/library/conditions.py +++ b/app/library/conditions.py @@ -180,7 +180,7 @@ class Conditions(metaclass=Singleton): if not isinstance(item, dict): if not isinstance(item, Condition): msg = f"Unexpected '{type(item).__name__}' item type." - raise ValueError(msg) # noqa: TRY004 + raise ValueError(msg) item = item.serialize() @@ -211,7 +211,7 @@ class Conditions(metaclass=Singleton): if not isinstance(item.get("extras"), dict): msg = "Extras must be a dictionary." - raise ValueError(msg) # noqa: TRY004 + raise ValueError(msg) return True diff --git a/app/library/config.py b/app/library/config.py index 65f5ac1e..25287879 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -27,6 +27,9 @@ class Config(metaclass=Singleton): app_env: str = "production" """The application environment, can be 'production' or 'development'.""" + app_path: str = "../../" + """The app path of the application.""" + config_path: str = "." """The path to the configuration directory.""" @@ -194,6 +197,7 @@ class Config(metaclass=Singleton): "temp_path", "config_path", "download_path", + "app_path", ) "The variables that are set manually." @@ -289,6 +293,7 @@ class Config(metaclass=Singleton): self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str( Path(baseDefaultPath) / "var" / "downloads" ) + self.app_path = Path(__file__).parent.parent.absolute() envFile: str = Path(self.config_path) / ".env" @@ -428,6 +433,21 @@ class Config(metaclass=Singleton): if "dev-master" == self.app_version: self._version_via_git() + def set_app_path(self, path: Path|str) -> "Config": + """ + Set the root path of the application. + + Args: + path (str): The root path to set. + + Returns: + Config: The Config instance. + + """ + Config.app_path = str(path) + + return self + def _get_attributes(self) -> dict: attrs: dict = {} vClass: str = self.__class__ diff --git a/app/library/dl_fields.py b/app/library/dl_fields.py index 85abee56..f901c4d3 100644 --- a/app/library/dl_fields.py +++ b/app/library/dl_fields.py @@ -228,7 +228,7 @@ class DLFields(metaclass=Singleton): if not isinstance(item, dict): if not isinstance(item, DLField): msg = f"Unexpected '{type(item).__name__}' type was given." - raise ValueError(msg) # noqa: TRY004 + raise ValueError(msg) item = item.serialize() @@ -255,7 +255,7 @@ class DLFields(metaclass=Singleton): if not isinstance(item.get("extras", {}), dict): msg = "Extras must be a dictionary." - raise ValueError(msg) # noqa: TRY004 + raise ValueError(msg) if re.match(r"^--[a-zA-Z0-9\-]+$", item.get("field", "").strip()) is None: msg = "Invalid yt-dlp option field it must starts with '--' and contain only alphanumeric characters." diff --git a/app/library/task_handlers/generic.py b/app/library/task_handlers/generic.py index 96d8d32e..78740fe9 100644 --- a/app/library/task_handlers/generic.py +++ b/app/library/task_handlers/generic.py @@ -607,6 +607,11 @@ class GenericTaskHandler(BaseHandler): cls._definitions = load_task_definitions() cls._sources_mtime = current + @classmethod + def refresh_definitions(cls, force: bool = False) -> None: + """Public helper to refresh cached task definitions.""" + cls._refresh_definitions(force=force) + @classmethod def _find_definition(cls, url: str) -> TaskDefinition | None: """ diff --git a/app/main.py b/app/main.py index f0500960..1a138d0f 100644 --- a/app/main.py +++ b/app/main.py @@ -28,6 +28,7 @@ from app.library.Notifications import Notification from app.library.Presets import Presets from app.library.Scheduler import Scheduler from app.library.Services import Services +from app.library.TaskDefinitions import TaskDefinitions from app.library.Tasks import Tasks LOG = logging.getLogger("app") @@ -39,6 +40,7 @@ ROOT_PATH: Path = Path(__file__).parent.absolute() class Main: def __init__(self, is_native: bool = False): self._config: Config = Config.get_instance(is_native=is_native) + self._config.set_app_path(str(ROOT_PATH)) self._app = web.Application() self._app.on_shutdown.append(self.on_shutdown) self._background_worker = BackgroundWorker() @@ -132,6 +134,7 @@ class Main: Notification.get_instance().attach(self._app) Conditions.get_instance().attach(self._app) DLFields.get_instance().attach(self._app) + TaskDefinitions.get_instance().attach(self._app) self._background_worker.attach(self._app) EventBus.get_instance().emit( diff --git a/app/routes/api/task_definitions.py b/app/routes/api/task_definitions.py new file mode 100644 index 00000000..bda8f8fb --- /dev/null +++ b/app/routes/api/task_definitions.py @@ -0,0 +1,145 @@ +import logging +from typing import Any + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.encoder import Encoder +from app.library.router import route +from app.library.TaskDefinitions import TaskDefinitionRecord, TaskDefinitions + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/task_definitions/", "task_definitions") +async def task_definitions_list(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: + include: str | None = request.query.get("include") + include_definition: bool = "definition" == include + + return web.json_response( + data=[item.serialize(include_definition=include_definition) for item in task_definitions.list()], + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("GET", "api/task_definitions/{identifier}", "task_definitions_get") +async def task_definitions_get(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: + identifier: str = request.match_info.get("identifier", "").strip() + if not identifier: + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + record: TaskDefinitionRecord | None = task_definitions.get(identifier) + if not record: + return web.json_response( + data={"error": f"Task definition '{identifier}' not found."}, + status=web.HTTPNotFound.status_code, + ) + + return web.json_response( + data=record.serialize(include_definition=True), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/task_definitions/", "task_definitions_create") +async def task_definitions_create(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: + try: + payload: Any = await request.json() + if not isinstance(payload, dict): + return web.json_response( + data={"error": "Invalid request body; expected JSON object."}, + status=web.HTTPBadRequest.status_code, + ) + + if "definition" in payload: + if not isinstance(payload["definition"], dict): + return web.json_response( + data={"error": "definition must be a JSON object when provided."}, + status=web.HTTPBadRequest.status_code, + ) + payload = payload["definition"] + + record: TaskDefinitionRecord = task_definitions.create(payload) + except (ValueError, TypeError) as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": "Failed to create task definition."}, + status=web.HTTPInternalServerError.status_code, + ) + + return web.json_response( + data=record.serialize(include_definition=True), + status=web.HTTPCreated.status_code, + dumps=encoder.encode, + ) + + +@route("PUT", "api/task_definitions/{identifier}", "task_definitions_update") +async def task_definitions_update(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: + identifier: str = request.match_info.get("identifier", "").strip() + if not identifier: + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + payload: Any = await request.json() + if not isinstance(payload, dict): + return web.json_response( + data={"error": "Invalid request body; expected JSON object."}, + status=web.HTTPBadRequest.status_code, + ) + + if "definition" in payload: + if not isinstance(payload["definition"], dict): + return web.json_response( + data={"error": "definition must be a JSON object when provided."}, + status=web.HTTPBadRequest.status_code, + ) + payload = payload["definition"] + + record: TaskDefinitionRecord = task_definitions.update(identifier, payload) + except (ValueError, TypeError) as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": "Failed to update task definition."}, + status=web.HTTPInternalServerError.status_code, + ) + + return web.json_response( + data=record.serialize(include_definition=True), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("DELETE", "api/task_definitions/{identifier}", "task_definitions_delete") +async def task_definitions_delete(request: Request, task_definitions: TaskDefinitions) -> Response: + identifier: str = request.match_info.get("identifier", "").strip() + if not identifier: + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + task_definitions.delete(identifier) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": "Failed to delete task definition."}, status=web.HTTPInternalServerError.status_code + ) + + return web.json_response(data={"status": "deleted"}, status=web.HTTPOk.status_code) diff --git a/app/schema/conditions.json b/app/schema/conditions.json new file mode 100644 index 00000000..54b4c2a0 --- /dev/null +++ b/app/schema/conditions.json @@ -0,0 +1,56 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://ytptube.app/schemas/conditions.json", + "title": "YTPTube Download Conditions", + "description": "Schema describing conditional rules evaluated against yt-dlp info dicts.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "filter" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the condition (UUID)." + }, + "name": { + "type": "string", + "description": "Human-readable name for the condition." + }, + "filter": { + "type": "string", + "description": "Filter expression to evaluate against info dict." + }, + "cli": { + "type": "string", + "description": "yt-dlp CLI fragment to append if matched.", + "default": "" + }, + "extras": { + "type": "object", + "description": "Any extra data to store with the condition.", + "default": {} + } + }, + "examples": [ + { + "id": "c1e2d3f4-5678-1234-9abc-def012345678", + "name": "Audio Only", + "filter": "type=audio", + "cli": "--extract-audio", + "extras": { + "priority": "high" + } + }, + { + "id": "a2b3c4d5-6789-2345-0bcd-ef1234567890", + "name": "Long Videos", + "filter": "duration>3600" + } + ] + } +} diff --git a/app/schema/dl_fields.json b/app/schema/dl_fields.json new file mode 100644 index 00000000..de458fb5 --- /dev/null +++ b/app/schema/dl_fields.json @@ -0,0 +1,78 @@ +{ + "$id": "https://ytptube.app/schema/dl_fields.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Download Field Definition", + "description": "Schema for custom yt-dlp field definitions used in YTPTube.", + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name", + "description", + "field", + "kind" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the field (UUID)." + }, + "name": { + "type": "string", + "description": "Human-readable name for the field." + }, + "description": { + "type": "string", + "description": "Description of the field." + }, + "field": { + "type": "string", + "description": "yt-dlp field or CLI argument." + }, + "kind": { + "type": "string", + "description": "Type of field (string, text, bool).", + "enum": [ + "string", + "text", + "bool" + ] + }, + "icon": { + "type": "string", + "description": "Font-awesome icon for the field.", + "default": "" + }, + "order": { + "type": "integer", + "description": "Order for sorting fields in the UI.", + "default": 0 + }, + "value": { + "type": "string", + "description": "Default value for the field.", + "default": "" + }, + "extras": { + "type": "object", + "description": "Additional options for the field.", + "default": {} + } + }, + "additionalProperties": false, + "examples": [ + { + "id": "acffe9f6-993b-42ad-94ff-4646f48a83a9", + "name": "Delete cache?", + "description": "Delete cache", + "field": "--no-continue", + "kind": "bool", + "icon": "fa-solid fa-trash-arrow-up", + "order": 1, + "value": "", + "extras": {} + } + ] + } +} diff --git a/app/schema/notifications.json b/app/schema/notifications.json new file mode 100644 index 00000000..e2f48a57 --- /dev/null +++ b/app/schema/notifications.json @@ -0,0 +1,175 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://ytptube.app/schema/notifications.json", + "title": "Notification Configuration", + "type": "array", + "description": "Schema describing webhook and Apprise notification targets stored in notifications.json.", + "items": { + "$ref": "#/definitions/target" + }, + "definitions": { + "event": { + "type": "string", + "enum": [ + "test", + "item_added", + "item_completed", + "item_cancelled", + "item_deleted", + "item_paused", + "item_resumed", + "item_moved", + "paused", + "resumed", + "log_info", + "log_success", + "log_warning", + "log_error", + "task_dispatched" + ], + "description": "Event identifier emitted by the backend event bus." + }, + "header": { + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "value" + ], + "properties": { + "key": { + "type": "string", + "minLength": 1, + "description": "HTTP header name." + }, + "value": { + "type": "string", + "minLength": 1, + "description": "HTTP header value." + } + } + }, + "request": { + "type": "object", + "additionalProperties": false, + "required": [ + "type", + "method", + "url" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "json", + "form" + ], + "default": "json", + "description": "Payload encoding strategy for the outgoing request." + }, + "method": { + "type": "string", + "enum": [ + "POST", + "PUT" + ], + "default": "POST", + "description": "HTTP method used when delivering the notification payload." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Destination endpoint or Apprise URL that receives the notification." + }, + "data_key": { + "type": "string", + "minLength": 1, + "default": "data", + "description": "JSON/Form field name used to wrap the serialized event payload." + }, + "headers": { + "type": "array", + "items": { + "$ref": "#/definitions/header" + }, + "description": "Optional static headers appended to each HTTP request.", + "default": [] + } + } + }, + "target": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "request" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid", + "description": "Unique identifier for the notification target (UUIDv4)." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Label shown in the UI for this notification target." + }, + "on": { + "type": "array", + "items": { + "$ref": "#/definitions/event" + }, + "description": "Subset of events that trigger this target; leave empty to listen to every event.", + "default": [] + }, + "presets": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "description": "Restrict notifications to downloads created by these presets.", + "default": [] + }, + "request": { + "$ref": "#/definitions/request" + } + }, + "examples": [ + { + "id": "d40d4b6e-45b8-47a1-b1b2-9cf7f8e6c67a", + "name": "Webhook: Completed items", + "on": [ + "item_completed" + ], + "presets": [ + "default" + ], + "request": { + "type": "json", + "method": "POST", + "url": "https://hooks.example.com/ytptube", + "headers": [ + { + "key": "Authorization", + "value": "Bearer token123" + } + ] + } + }, + { + "id": "9f783d0a-6e8b-4c1d-9089-5bb4ae0f2c47", + "name": "Apprise Notifier", + "on": [], + "request": { + "type": "json", + "method": "POST", + "url": "discord://webhook/token" + } + } + ] + } + } +} diff --git a/app/schema/presets.json b/app/schema/presets.json new file mode 100644 index 00000000..a751d97c --- /dev/null +++ b/app/schema/presets.json @@ -0,0 +1,67 @@ +{ + "$id": "https://ytptube.app/schema/presets.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Preset Configuration", + "description": "Schema for yt-dlp preset configurations used in YTPTube.", + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the preset (UUID or slug)." + }, + "name": { + "type": "string", + "description": "Human-readable name for the preset.", + "minLength": 1 + }, + "description": { + "type": "string", + "description": "Optional description of the preset.", + "default": "" + }, + "folder": { + "type": "string", + "description": "Default download folder for this preset.", + "default": "" + }, + "template": { + "type": "string", + "description": "Default output template for this preset.", + "default": "" + }, + "cookies": { + "type": "string", + "description": "Default cookies for this preset.", + "default": "" + }, + "cli": { + "type": "string", + "description": "yt-dlp command line options for this preset.", + "default": "" + }, + "default": { + "type": "boolean", + "description": "Whether this preset is a default preset.", + "default": false + } + }, + "required": [ + "id", + "name" + ], + "examples": [ + { + "id": "3e163c6c-64eb-4448-924f-814b629b3810", + "name": "default", + "default": true, + "cli": "--socket-timeout 30 --download-archive %(config_path)s/archive.log", + "description": "Default preset for yt-dlp. It will download whatever yt-dlp decides is the best quality for the video and audio." + }, + { + "id": "a6fd4b25-2b3e-458d-bb57-b75e41cc4330", + "name": "Audio Only", + "cli": "--extract-audio --add-chapters --embed-metadata --embed-thumbnail --format 'bestaudio/best'", + "description": "This preset is designed to download only the audio of the video. It will extract the audio, add chapters, metadata, and thumbnail." + } + ] +} diff --git a/app/library/task_handlers/task_definition.schema.json b/app/schema/task_definition.json similarity index 93% rename from app/library/task_handlers/task_definition.schema.json rename to app/schema/task_definition.json index 711b06d3..1e3c8917 100644 --- a/app/library/task_handlers/task_definition.schema.json +++ b/app/schema/task_definition.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://ytptube.app/schemas/task-definition.json", + "$id": "https://ytptube.app/schemas/task_definition.json", "title": "YTPTube Generic Task Definition", "type": "object", "required": [ @@ -15,6 +15,12 @@ "minLength": 1, "description": "Human-readable identifier for this definition." }, + "priority": { + "type": "integer", + "minimum": 0, + "description": "Optional ordering priority. Lower numbers are listed first.", + "default": 0 + }, "match": { "type": "array", "minItems": 1, @@ -226,6 +232,34 @@ ] } }, + "allOf": [ + { + "if": { + "properties": { + "engine": { + "properties": { + "type": { "const": "selenium" } + }, + "required": ["type"] + } + }, + "required": ["engine"] + }, + "then": { + "properties": { + "engine": { + "properties": { + "options": { + "required": ["url"] + } + }, + "required": ["options"] + } + }, + "required": ["engine"] + } + } + ], "definitions": { "stringMap": { "type": "object", diff --git a/app/schema/tasks.json b/app/schema/tasks.json new file mode 100644 index 00000000..74396c7e --- /dev/null +++ b/app/schema/tasks.json @@ -0,0 +1,84 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://ytptube.app/schemas/tasks.json", + "title": "YTPTube Scheduled Tasks", + "description": "Schema describing recurring download tasks stored in tasks.json.", + "type": "array", + "items": { + "$ref": "#/definitions/task" + }, + "definitions": { + "task": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "url" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid", + "description": "Unique identifier for the task (UUIDv4)." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-friendly task label used throughout the UI." + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://", + "description": "Playlist or channel URL monitored by the task." + }, + "folder": { + "type": "string", + "description": "Optional download folder override for this task.", + "default": "" + }, + "preset": { + "type": "string", + "description": "Preset name to apply; falls back to the global default when omitted.", + "default": "" + }, + "timer": { + "type": "string", + "description": "Cron expression executed by the scheduler (five or six space-separated fields).", + "default": "" + }, + "template": { + "type": "string", + "description": "Output filename template override for downloads created by this task.", + "default": "" + }, + "cli": { + "type": "string", + "description": "Additional yt-dlp arguments appended to the download request.", + "default": "" + }, + "auto_start": { + "type": "boolean", + "description": "Automatically queue new items discovered by the task.", + "default": true + }, + "handler_enabled": { + "type": "boolean", + "description": "Controls whether the matched handler remains enabled for this task.", + "default": true + } + }, + "examples": [ + { + "id": "4d9be675-ecb8-4b42-9a84-8cee71a7a2e2", + "name": "Weekly playlist sync", + "url": "https://www.youtube.com/playlist?list=PL12345", + "preset": "default", + "timer": "0 */6 * * *", + "auto_start": true + } + ] + } + } +} diff --git a/app/tests/test_task_definitions.py b/app/tests/test_task_definitions.py new file mode 100644 index 00000000..28a42bb8 --- /dev/null +++ b/app/tests/test_task_definitions.py @@ -0,0 +1,315 @@ +import json +import uuid +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +import pytest +from aiohttp import web +from aiohttp.web import Request +from jsonschema import Draft7Validator + +from app.library.encoder import Encoder +from app.library.TaskDefinitions import TaskDefinitionRecord, TaskDefinitions +from app.routes.api import task_definitions as api + + +def _load_validator() -> Draft7Validator: + schema_path = Path(__file__).resolve().parent.parent / "schema" / "task_definition.json" + schema = json.loads(schema_path.read_text(encoding="utf-8")) + return Draft7Validator(schema) + + +def _sample_definition(name: str = "example", *, priority: int = 0) -> dict[str, Any]: + return { + "name": name, + "match": ["https://example.com/*"], + "priority": priority, + "parse": { + "items": { + "type": "css", + "selector": ".card", + "fields": { + "link": {"type": "css", "expression": "a", "attribute": "href"}, + "title": {"type": "css", "expression": "a", "attribute": "text"}, + }, + } + }, + } + + +class TestTaskDefinitionsManager: + def setup_method(self) -> None: + TaskDefinitions._reset_singleton() + + def teardown_method(self) -> None: + TaskDefinitions._reset_singleton() + + def test_load_populates_records(self, tmp_path: Path) -> None: + validator = _load_validator() + config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) + + first_identifier = "b5c6ad5f-4745-4c05-88c8-dde1deae3b51" + second_identifier = "ae38a6b0-2c22-4763-ba60-801ae8ce1218" + + first = tmp_path / f"{first_identifier}.json" + second = tmp_path / f"{second_identifier}.json" + + first.write_text(json.dumps(_sample_definition("First", priority=5)), encoding="utf-8") + second.write_text(json.dumps(_sample_definition("Second", priority=1)), encoding="utf-8") + + (tmp_path / "not-a-uuid.json").write_text(json.dumps(_sample_definition("Ignored")), encoding="utf-8") + (tmp_path / "0f9184de-5d3c-2111-8c21-6d3f0be1bd3d.json").write_text( + json.dumps(_sample_definition("Ignored2")), + encoding="utf-8", + ) + + manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) + + records = manager.list() + assert len(records) == 2 + assert [record.name for record in records] == ["Second", "First"] + assert [record.priority for record in records] == [1, 5] + + def test_create_writes_file_and_refreshes(self, tmp_path: Path) -> None: + validator = _load_validator() + config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) + manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) + + with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh: + definition = _sample_definition("My Definition") + record = manager.create(definition) + + refresh.assert_called_once_with(force=True) + identifier_uuid = uuid.UUID(record.identifier) + assert 4 == identifier_uuid.version + expected_filename = f"{record.identifier}.json" + saved_path = tmp_path / expected_filename + assert saved_path.exists() + saved_content = json.loads(saved_path.read_text(encoding="utf-8")) + assert saved_content["name"] == "My Definition" + assert saved_content["priority"] == 0 + + def test_update_missing_definition_raises(self, tmp_path: Path) -> None: + validator = _load_validator() + config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) + manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) + + with pytest.raises(ValueError, match="does not exist"): + manager.update("missing", _sample_definition("Updated")) + + def test_update_overwrites_file(self, tmp_path: Path) -> None: + validator = _load_validator() + config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) + + identifier = "c59ec7cf-6291-4f0f-86f8-d8cb12c325a4" + initial = _sample_definition("Original", priority=4) + path = tmp_path / f"{identifier}.json" + path.write_text(json.dumps(initial), encoding="utf-8") + + manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) + + with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh: + updated_record = manager.update(identifier, _sample_definition("Updated", priority=2)) + + refresh.assert_called_once_with(force=True) + assert updated_record.name == "Updated" + assert updated_record.priority == 2 + saved = json.loads(path.read_text(encoding="utf-8")) + assert saved["name"] == "Updated" + assert saved["priority"] == 2 + + def test_delete_removes_file(self, tmp_path: Path) -> None: + validator = _load_validator() + config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) + + identifier = "f0b71f47-6b65-4b6d-89fd-6b87ce47d3bc" + definition_path = tmp_path / f"{identifier}.json" + definition_path.write_text(json.dumps(_sample_definition("Delete")), encoding="utf-8") + + manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) + + with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh: + manager.delete(identifier) + + refresh.assert_called_once_with(force=True) + assert not definition_path.exists() + assert manager.get(identifier) is None + + +@pytest.mark.asyncio +class TestTaskDefinitionRoutes: + def setup_method(self) -> None: + TaskDefinitions._reset_singleton() + + def teardown_method(self) -> None: + TaskDefinitions._reset_singleton() + + async def test_list_definitions(self) -> None: + request = MagicMock(spec=Request) + request.query = {} + + identifier = "9af7018f-8659-4d2a-a42b-b5d2c5f0a6e2" + record = TaskDefinitionRecord( + identifier=identifier, + filename=f"{identifier}.json", + name="Sample", + priority=0, + path=Path("/tmp/sample.json"), + data=_sample_definition("Sample"), + updated_at=123.0, + ) + + task_definitions = MagicMock(spec=TaskDefinitions) + task_definitions.list.return_value = [record] + + response = await api.task_definitions_list(request, Encoder(), task_definitions) + payload = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code + assert payload == [record.serialize()] + assert "filename" not in payload[0] + + async def test_list_definitions_includes_definition(self) -> None: + request = MagicMock(spec=Request) + request.query = {"include": "definition"} + + identifier = "f5b5e88d-5c6b-4a27-8453-1b6a4fb8a8d1" + record = TaskDefinitionRecord( + identifier=identifier, + filename=f"{identifier}.json", + name="Sample", + priority=0, + path=Path("/tmp/sample.json"), + data=_sample_definition("Sample"), + updated_at=123.0, + ) + + task_definitions = MagicMock(spec=TaskDefinitions) + task_definitions.list.return_value = [record] + + response = await api.task_definitions_list(request, Encoder(), task_definitions) + payload = json.loads(response.text) + + assert payload[0]["definition"]["name"] == "Sample" + + async def test_get_definition_not_found(self) -> None: + request = MagicMock(spec=Request) + request.match_info = {"identifier": "unknown"} + + task_definitions = MagicMock(spec=TaskDefinitions) + task_definitions.get.return_value = None + + response = await api.task_definitions_get(request, Encoder(), task_definitions) + payload = json.loads(response.text) + + assert response.status == web.HTTPNotFound.status_code + assert "error" in payload + + async def test_create_definition_success(self) -> None: + payload_definition = _sample_definition("New", priority=1) + payload = {"definition": payload_definition} + + request = MagicMock(spec=Request) + request.json = AsyncMock(return_value=payload) + + identifier = "4f08b8af-b87a-4d6e-9289-39e5172898aa" + record = TaskDefinitionRecord( + identifier=identifier, + filename=f"{identifier}.json", + name="New", + priority=1, + path=Path("/tmp/new.json"), + data=payload["definition"], + updated_at=123.0, + ) + + task_definitions = MagicMock(spec=TaskDefinitions) + task_definitions.create.return_value = record + + response = await api.task_definitions_create(request, Encoder(), task_definitions) + body = json.loads(response.text) + + assert response.status == web.HTTPCreated.status_code + assert body["id"] == identifier + assert body["priority"] == 1 + assert "filename" not in body + task_definitions.create.assert_called_once_with(payload_definition) + + async def test_create_definition_invalid_payload(self) -> None: + request = MagicMock(spec=Request) + request.json = AsyncMock(return_value=[]) # type: ignore[arg-type] + + task_definitions = MagicMock(spec=TaskDefinitions) + + response = await api.task_definitions_create(request, Encoder(), task_definitions) + body = json.loads(response.text) + + assert response.status == web.HTTPBadRequest.status_code + assert "error" in body + + async def test_update_definition_success(self) -> None: + request = MagicMock(spec=Request) + identifier = "6d8d5719-95ae-4478-bb05-986f5b72b6c1" + request.match_info = {"identifier": identifier} + definition = _sample_definition("Updated", priority=4) + request.json = AsyncMock(return_value={"definition": definition}) + record = TaskDefinitionRecord( + identifier=identifier, + filename=f"{identifier}.json", + name="Updated", + priority=4, + path=Path("/tmp/existing.json"), + data=definition, + updated_at=456.0, + ) + + task_definitions = MagicMock(spec=TaskDefinitions) + task_definitions.update.return_value = record + + response = await api.task_definitions_update(request, Encoder(), task_definitions) + body = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code + assert body["name"] == "Updated" + assert body["priority"] == 4 + task_definitions.update.assert_called_once_with(identifier, definition) + + async def test_update_definition_missing_identifier(self) -> None: + request = MagicMock(spec=Request) + request.match_info = {"identifier": ""} + request.json = AsyncMock(return_value={}) + + task_definitions = MagicMock(spec=TaskDefinitions) + + response = await api.task_definitions_update(request, Encoder(), task_definitions) + body = json.loads(response.text) + + assert response.status == web.HTTPBadRequest.status_code + assert "error" in body + + async def test_delete_definition_success(self) -> None: + request = MagicMock(spec=Request) + identifier = "c9f4ac6c-a4ab-4d1a-8d25-764dc0c8a3f0" + request.match_info = {"identifier": identifier} + + task_definitions = MagicMock(spec=TaskDefinitions) + + response = await api.task_definitions_delete(request, task_definitions) + body = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code + assert body["status"] == "deleted" + task_definitions.delete.assert_called_once_with(identifier) + + async def test_delete_definition_missing_identifier(self) -> None: + request = MagicMock(spec=Request) + request.match_info = {"identifier": ""} + + task_definitions = MagicMock(spec=TaskDefinitions) + + response = await api.task_definitions_delete(request, task_definitions) + body = json.loads(response.text) + + assert response.status == web.HTTPBadRequest.status_code + assert "error" in body diff --git a/pyproject.toml b/pyproject.toml index 534abeaa..bf58a242 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ "selenium>=4.35.0", "parsel>=1.10.0", "jmespath>=1.0.1", + "jsonschema>=4.23.0", ] [tool.ruff] @@ -164,6 +165,8 @@ ignore = [ "PLC0415", "S603", "D203", + "TRY004", # Like it's our choice to use ValuesError :| + "PT011", ] # Allow fix for all enabled rules (when `--fix`) is provided. diff --git a/ui/app/components/TaskDefinitionEditor.vue b/ui/app/components/TaskDefinitionEditor.vue new file mode 100644 index 00000000..7d6de469 --- /dev/null +++ b/ui/app/components/TaskDefinitionEditor.vue @@ -0,0 +1,870 @@ +