diff --git a/API.md b/API.md index fc477369..7ff34e34 100644 --- a/API.md +++ b/API.md @@ -42,11 +42,12 @@ This document describes the available endpoints and their usage. All endpoints r - [DELETE /api/tasks/{id}/mark](#delete-apitasksidmark) - [POST /api/tasks/{id}/metadata](#post-apitasksidmetadata) - [PATCH /api/tasks/{id}](#patch-apitasksid) - - [GET /api/task\_definitions/](#get-apitask_definitions) - - [GET /api/task\_definitions/{identifier}](#get-apitask_definitionsidentifier) - - [POST /api/task\_definitions/](#post-apitask_definitions) - - [PUT /api/task\_definitions/{identifier}](#put-apitask_definitionsidentifier) - - [DELETE /api/task\_definitions/{identifier}](#delete-apitask_definitionsidentifier) + - [GET /api/tasks/definitions/](#get-apitasksdefinitions) + - [GET /api/tasks/definitions/{id}](#get-apitasksdefinitionsid) + - [POST /api/tasks/definitions/](#post-apitasksdefinitions) + - [PUT /api/tasks/definitions/{id}](#put-apitasksdefinitionsid) + - [PATCH /api/tasks/definitions/{id}](#patch-apitasksdefinitionsid) + - [DELETE /api/tasks/definitions/{id}](#delete-apitasksdefinitionsid) - [GET /api/player/playlist/{file:.\*}.m3u8](#get-apiplayerplaylistfilem3u8) - [GET /api/player/m3u8/{mode}/{file:.\*}.m3u8](#get-apiplayerm3u8modefilem3u8) - [GET /api/player/segments/{segment}/{file:.\*}.ts](#get-apiplayersegmentssegmentfilets) @@ -70,11 +71,11 @@ This document describes the available endpoints and their usage. All endpoints r - [PUT /api/presets](#put-apipresets) - [GET /api/conditions/](#get-apiconditions) - [POST /api/conditions/](#post-apiconditions) + - [POST /api/conditions/test](#post-apiconditionstest) - [GET /api/conditions/{id}](#get-apiconditionsid) - [PATCH /api/conditions/{id}](#patch-apiconditionsid) - [PUT /api/conditions/{id}](#put-apiconditionsid) - [DELETE /api/conditions/{id}](#delete-apiconditionsid) - - [POST /api/conditions/test](#post-apiconditionstest) - [GET /api/logs](#get-apilogs) - [GET /api/logs/stream](#get-apilogsstream) - [GET /api/notifications/](#get-apinotifications) @@ -1140,54 +1141,71 @@ Returns the updated task --- -### GET /api/task_definitions/ -**Purpose**: Retrieve all task definitions. +### GET /api/tasks/definitions/ +**Purpose**: Retrieve task definitions. **Query Parameters**: - `include=definition` (optional) - Include the full definition object in response. - -**Response**: -```json -[ - { - "id": "", - "name": "Task Definition Name", - "description": "...", - "enabled": true, - "definition": { ... } // only if include=definition - }, - ... -] -``` - ---- - -### GET /api/task_definitions/{identifier} -**Purpose**: Retrieve a specific task definition by ID or name. - -**Path Parameter**: -- `identifier`: Task definition ID or name. +- `page` (optional): Page number (1-indexed). Default: `1`. +- `per_page` (optional): Items per page. Default: `config.default_pagination`. **Response**: ```json { - "id": "", - "name": "Task Definition Name", - "description": "...", - "enabled": true, - "definition": { - "handler": "GenericTaskHandler", - "config": { ... } + "items": [ + { + "id": 1, + "name": "Task Definition Name", + "description": "...", + "enabled": true, + "definition": { ... } // only if include=definition + }, + ... + ], + "pagination": { + "page": 1, + "per_page": 50, + "total": 1, + "total_pages": 1, + "has_next": false, + "has_prev": false } } ``` -- `400 Bad Request` if identifier is missing. +--- + +### GET /api/tasks/definitions/{id} +**Purpose**: Retrieve a specific task definition by ID. + +**Path Parameter**: +- `id`: Task definition ID. + +**Response**: +```json +{ + "id": 1, + "name": "Task Definition Name", + "description": "...", + "enabled": true, + "definition": { + "parse": { + "url": { ... }, + "items": { ... } + }, + "engine": { ... }, + "request": { ... }, + "response": { ... } + } +} +``` + +- `400 Bad Request` if ID is missing. - `404 Not Found` if the task definition doesn't exist. --- -### POST /api/task_definitions/ +### POST /api/tasks/definitions/ **Purpose**: Create a new task definition. **Body**: @@ -1197,19 +1215,13 @@ Returns the updated task "description": "...", "enabled": true, "definition": { - "handler": "GenericTaskHandler", - "config": { ... } - } -} -``` - -Or wrap in a definition object: -```json -{ - "definition": { - "name": "My Task Definition", - "handler": "GenericTaskHandler", - ... + "parse": { + "url": { ... }, + "items": { ... } + }, + "engine": { ... }, + "request": { ... }, + "response": { ... } } } ``` @@ -1217,7 +1229,7 @@ Or wrap in a definition object: **Response**: ```json { - "id": "", + "id": 1, "name": "My Task Definition", "description": "...", "enabled": true, @@ -1230,11 +1242,11 @@ Or wrap in a definition object: --- -### PUT /api/task_definitions/{identifier} -**Purpose**: Update an existing task definition. +### PUT /api/tasks/definitions/{id} +**Purpose**: Replace an existing task definition. **Path Parameter**: -- `identifier`: Task definition ID or name. +- `id`: Task definition ID. **Body**: ```json @@ -1243,8 +1255,10 @@ Or wrap in a definition object: "description": "...", "enabled": false, "definition": { - "handler": "GenericTaskHandler", - "config": { ... } + "parse": { ... }, + "engine": { ... }, + "request": { ... }, + "response": { ... } } } ``` @@ -1252,7 +1266,7 @@ Or wrap in a definition object: **Response**: ```json { - "id": "", + "id": 1, "name": "Updated Name", "description": "...", "enabled": false, @@ -1261,23 +1275,53 @@ Or wrap in a definition object: ``` - `200 OK` if successful. -- `400 Bad Request` if identifier is missing or validation fails. +- `400 Bad Request` if ID is missing or validation fails. +- `404 Not Found` if the task definition doesn't exist. --- -### DELETE /api/task_definitions/{identifier} +### PATCH /api/tasks/definitions/{id} +**Purpose**: Partially update a task definition. + +**Path Parameter**: +- `id`: Task definition ID. + +**Body**: +```json +{ + "enabled": false, + "description": "Updated description" +} +``` + +**Response**: Updated task definition object. + +- `200 OK` if successful. +- `400 Bad Request` if ID is missing or validation fails. +- `404 Not Found` if the task definition doesn't exist. + +--- + +### DELETE /api/tasks/definitions/{id} **Purpose**: Delete a task definition. **Path Parameter**: -- `identifier`: Task definition ID or name. +- `id`: Task definition ID. **Response**: ```json -{ "status": "deleted" } +{ + "id": 1, + "name": "Deleted Definition", + "description": "...", + "enabled": false, + "definition": { ... } +} ``` - `200 OK` if successful. -- `400 Bad Request` if identifier is missing or task definition doesn't exist. +- `400 Bad Request` if ID is missing. +- `404 Not Found` if the task definition doesn't exist. --- @@ -1616,6 +1660,9 @@ Binary image data with appropriate headers ### GET /api/dl_fields/{id} **Purpose**: Retrieve a single download field by ID. +**Path Parameter**: +- `id`: Download field ID. + **Response**: ```json { "id": 1, "name": "Title", "description": "...", "field": "title", "kind": "text", "order": 0, "value": "", "icon": "fa-solid fa-tag", "extras": {} } @@ -1626,6 +1673,9 @@ Binary image data with appropriate headers ### PATCH /api/dl_fields/{id} **Purpose**: Partially update a download field. +**Path Parameter**: +- `id`: Download field ID. + **Body**: ```json { "description": "Updated description", "order": 1 } @@ -1638,6 +1688,9 @@ Binary image data with appropriate headers ### PUT /api/dl_fields/{id} **Purpose**: Replace a download field. +**Path Parameter**: +- `id`: Download field ID. + **Body**: ```json { @@ -1659,6 +1712,9 @@ Binary image data with appropriate headers ### DELETE /api/dl_fields/{id} **Purpose**: Delete a download field by ID. +**Path Parameter**: +- `id`: Download field ID. + **Response**: ```json { "id": 1, "name": "Title", "description": "...", "field": "title", "kind": "text", "order": 0, "value": "", "icon": "fa-solid fa-tag", "extras": {} } @@ -1759,41 +1815,6 @@ Binary image data with appropriate headers --- -### GET /api/conditions/{id} -**Purpose**: Retrieve a condition by ID. - -**Response**: Condition object. - ---- - -### PATCH /api/conditions/{id} -**Purpose**: Partially update a condition. - -**Body**: -```json -{ "enabled": false, "priority": 5 } -``` - -**Response**: Updated condition object. - ---- - -### PUT /api/conditions/{id} -**Purpose**: Replace a condition. - -**Body**: Full condition object. - -**Response**: Updated condition object. - ---- - -### DELETE /api/conditions/{id} -**Purpose**: Delete a condition by ID. - -**Response**: Deleted condition object. - ---- - ### POST /api/conditions/test **Purpose**: Evaluate a condition expression against info extracted from a URL. @@ -1815,6 +1836,53 @@ Binary image data with appropriate headers --- +### GET /api/conditions/{id} +**Purpose**: Retrieve a condition by ID. + +**Path Parameter**: +- `id`: Condition ID. + +**Response**: Condition object. + +--- + +### PATCH /api/conditions/{id} +**Purpose**: Partially update a condition. + +**Path Parameter**: +- `id`: Condition ID. + +**Body**: +```json +{ "enabled": false, "priority": 5 } +``` + +**Response**: Updated condition object. + +--- + +### PUT /api/conditions/{id} +**Purpose**: Replace a condition. + +**Path Parameter**: +- `id`: Condition ID. + +**Body**: Full condition object. + +**Response**: Updated condition object. + +--- + +### DELETE /api/conditions/{id} +**Purpose**: Delete a condition by ID. + +**Path Parameter**: +- `id`: Condition ID. + +**Response**: Deleted condition object. + +--- + ### GET /api/logs **Purpose**: Retrieve recent application logs (if file logging is enabled). @@ -1946,6 +2014,9 @@ Binary image data with appropriate headers ### GET /api/notifications/{id} **Purpose**: Retrieve a notification target by ID. +**Path Parameter**: +- `id`: Notification target ID. + **Response**: Notification target object. --- @@ -1953,6 +2024,9 @@ Binary image data with appropriate headers ### PATCH /api/notifications/{id} **Purpose**: Partially update a notification target. +**Path Parameter**: +- `id`: Notification target ID. + **Body**: ```json { "enabled": false } @@ -1965,6 +2039,9 @@ Binary image data with appropriate headers ### PUT /api/notifications/{id} **Purpose**: Replace a notification target. +**Path Parameter**: +- `id`: Notification target ID. + **Body**: Full notification target object. **Response**: Updated notification target. @@ -1974,6 +2051,9 @@ Binary image data with appropriate headers ### DELETE /api/notifications/{id} **Purpose**: Delete a notification target. +**Path Parameter**: +- `id`: Notification target ID. + **Response**: Deleted notification target. --- @@ -2186,7 +2266,7 @@ The WebSocket API provides real-time bidirectional communication between the cli ### Connection -**URL**: `ws://localhost:8081/ws` (development) or `wss://yourdomain.com/ws` (production) +**URL**: `ws://localhost:8081/ws` (development) or `wss://domain.example/ws` (production) The client automatically connects to the WebSocket server and receives a `connected` event with initial state. The frontend wrapper handles reconnection (default: up to 50 attempts, 5s delay). diff --git a/FAQ.md b/FAQ.md index fd034fc6..f1332746 100644 --- a/FAQ.md +++ b/FAQ.md @@ -171,25 +171,25 @@ Then restart the container to apply the changes. # How can I monitor sites without RSS feeds? -YTPTube includes a **generic task handler** that turns JSON definition files into site-specific scrapers. You can use it +YTPTube includes a **generic task handler** that turns JSON definitions into site-specific scrapers. You can use it to watch pages that do not expose RSS or public APIs and automatically enqueue new links into the download queue. -1. Create definition files under `/config/tasks/*.json` (for Docker this is the mounted `config/tasks/` folder). -2. Keep your scheduled task in `tasks.json` pointing at the page you want to monitor and make sure it uses a preset that - enables a download archive (`--download-archive`). -3. When the task runs, the handler scans the JSON files, picks the first definition whose `match` rule covers the task - URL, fetches the page, extracts items, and queues the unseen ones. +1. Create definition via the WebUI > tasks > Definitions. +2. Create task that reference same url click on inspect to see the results. Make sure it uses a preset that enables + a download archive (`--download-archive`). +3. When the task handler run, the handler scans the definitions, picks the first definition whose `match` rule covers + the task URL, fetches the page, extracts items, and queues the unseen ones. ### Definition schema -Each file must contain a single JSON object with the following keys: +Each definition must contain a single JSON object with the following keys: ```json5 { "name": "example", // Friendly identifier shown in logs - "match": [ - "https://example.com/articles/*", // Glob strings, or objects with {"regex": "..."} or {"glob": "..."} - { "regex": "https://example.com/post/[0-9]+" } + "match_url": [ + "https://example.com/articles/*", // Glob strings + "https://example.com/post/[0-9]+" // Regex strings ], "engine": { // Optional, defaults to HTTPX "type": "httpx", // "httpx" (default) or "selenium" @@ -207,7 +207,7 @@ Each file must contain a single JSON object with the following keys: "headers": { "User-Agent": "MyAgent/1.0" }, "params": { "page": 1 }, "data": null, - "json": null, + "json_data": null, "timeout": 30 }, "response": { // Optional: how to interpret the body @@ -247,6 +247,7 @@ For JSON endpoints, switch the response format and use `jsonpath` selectors: ```json5 { + ... "response": { "type": "json" }, "parse": { "items": { @@ -282,9 +283,6 @@ For JSON endpoints, switch the response format and use `jsonpath` selectors: the moment. Optional keys: `arguments` (list or string), `wait_for` (type `css`/`xpath` + `expression`), `wait_timeout`, and `page_load_timeout`. -Definitions are reloaded automatically when files change, so you can tweak them without restarting YTPTube. Check -`var/config/tasks/01-*.json` for sample files. - > [!NOTE] > A machine-readable schema is available at `app/schema/task_definition.json` if you want to validate your JSON with editors or CI tools. diff --git a/app/features/conditions/repository.py b/app/features/conditions/repository.py index 32def7dd..069987cf 100644 --- a/app/features/conditions/repository.py +++ b/app/features/conditions/repository.py @@ -120,13 +120,13 @@ class ConditionsRepository(metaclass=Singleton): clause: ColumnElement[bool] = ConditionModel.name == identifier result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1)) - model: ConditionModel | None = result.scalar_one_or_none() - - if not model: + if not (model := result.scalar_one_or_none()): msg: str = f"Condition '{identifier}' not found." raise KeyError(msg) payload.pop("id", None) + payload.pop("created_at", None) + payload.pop("updated_at", None) if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None: msg: str = f"Condition with name '{payload['name']}' already exists." @@ -142,7 +142,6 @@ class ConditionsRepository(metaclass=Singleton): async def delete(self, identifier: int | str) -> ConditionModel: async with self.session() as session: - # Query within this session if isinstance(identifier, int): clause: ColumnElement[bool] = ConditionModel.id == identifier elif isinstance(identifier, str) and identifier.isdigit(): @@ -153,9 +152,7 @@ class ConditionsRepository(metaclass=Singleton): clause: ColumnElement[bool] = ConditionModel.name == identifier result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1)) - model = result.scalar_one_or_none() - - if not model: + if not (model := result.scalar_one_or_none()): msg: str = f"Condition '{identifier}' not found." raise KeyError(msg) diff --git a/app/features/conditions/router.py b/app/features/conditions/router.py index b6fbf4f5..b432c82f 100644 --- a/app/features/conditions/router.py +++ b/app/features/conditions/router.py @@ -177,7 +177,7 @@ async def conditions_add(request: Request, encoder: Encoder, notify: EventBus) - return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode) -@route("GET", "api/conditions/{id}", name="condition_get") +@route("GET", r"api/conditions/{id:\d+}", name="condition_get") async def conditions_get(request: Request, encoder: Encoder) -> Response: """ Get the conditions @@ -199,7 +199,7 @@ async def conditions_get(request: Request, encoder: Encoder) -> Response: return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode) -@route("DELETE", "api/conditions/{id}", name="condition_delete") +@route("DELETE", r"api/conditions/{id:\d+}", name="condition_delete") async def conditions_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response: """ Delete Condition. @@ -226,7 +226,7 @@ async def conditions_delete(request: Request, encoder: Encoder, notify: EventBus return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) -@route("PATCH", "api/conditions/{id}", name="condition_patch") +@route("PATCH", r"api/conditions/{id:\d+}", name="condition_patch") async def conditions_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response: """ Patch Condition. @@ -277,7 +277,7 @@ async def conditions_patch(request: Request, encoder: Encoder, notify: EventBus) return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode) -@route("PUT", "api/conditions/{id}", name="condition_update") +@route("PUT", r"api/conditions/{id:\d+}", name="condition_update") async def conditions_update(request: Request, encoder: Encoder, notify: EventBus) -> Response: """ Update Condition. diff --git a/app/features/core/migration.py b/app/features/core/migration.py index db528037..a323e21a 100644 --- a/app/features/core/migration.py +++ b/app/features/core/migration.py @@ -2,7 +2,6 @@ from __future__ import annotations import abc import logging -import shutil import time from pathlib import Path from typing import TYPE_CHECKING @@ -46,7 +45,7 @@ class Migration(abc.ABC): timestamp = int(time.time()) destination: Path = self._migrated_dir / f"{source.stem}_{timestamp}{source.suffix}" - shutil.move(str(source), str(destination)) + source.rename(destination) return destination def _unique_name(self, name: str, seen_names: dict[str, int]) -> str: diff --git a/app/features/core/schemas.py b/app/features/core/schemas.py index ce2e8378..03ff776b 100644 --- a/app/features/core/schemas.py +++ b/app/features/core/schemas.py @@ -20,6 +20,7 @@ class CEFeature(str, Enum): DL_FIELDS = "dl_fields" CONDITIONS = "conditions" NOTIFICATIONS = "notifications" + TASKS_DEFINITIONS = "tasks_definitions" def __str__(self) -> str: return self.value diff --git a/app/features/dl_fields/router.py b/app/features/dl_fields/router.py index 41c8a7c6..330ea91c 100644 --- a/app/features/dl_fields/router.py +++ b/app/features/dl_fields/router.py @@ -67,7 +67,7 @@ async def dl_fields_add(request: Request, encoder: Encoder, notify: EventBus) -> return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode) -@route("GET", "api/dl_fields/{id}", "dl_fields_get") +@route("GET", r"api/dl_fields/{id:\d+}", "dl_fields_get") async def dl_fields_get(request: Request, encoder: Encoder) -> Response: if not (identifier := request.match_info.get("id")): return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) @@ -78,7 +78,7 @@ async def dl_fields_get(request: Request, encoder: Encoder) -> Response: return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode) -@route("DELETE", "api/dl_fields/{id}", "dl_fields_delete") +@route("DELETE", r"api/dl_fields/{id:\d+}", "dl_fields_delete") async def dl_fields_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response: if not (identifier := request.match_info.get("id")): return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) @@ -97,7 +97,7 @@ async def dl_fields_delete(request: Request, encoder: Encoder, notify: EventBus) return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) -@route("PATCH", "api/dl_fields/{id}", "dl_fields_patch") +@route("PATCH", r"api/dl_fields/{id:\d+}", "dl_fields_patch") async def dl_fields_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response: if not (identifier := request.match_info.get("id")): return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) @@ -138,7 +138,7 @@ async def dl_fields_patch(request: Request, encoder: Encoder, notify: EventBus) ) -@route("PUT", "api/dl_fields/{id}", "dl_fields_update") +@route("PUT", r"api/dl_fields/{id:\d+}", "dl_fields_update") async def dl_fields_update(request: Request, encoder: Encoder, notify: EventBus) -> Response: if not (identifier := request.match_info.get("id")): return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) diff --git a/app/features/notifications/router.py b/app/features/notifications/router.py index b357eb7e..bb5fdbf5 100644 --- a/app/features/notifications/router.py +++ b/app/features/notifications/router.py @@ -83,7 +83,7 @@ async def notifications_add(request: Request, encoder: Encoder, notify: EventBus return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode) -@route("GET", "api/notifications/{id}", "notification_get") +@route("GET", r"api/notifications/{id:\d+}", "notification_get") async def notifications_get(request: Request, encoder: Encoder) -> Response: if not (identifier := request.match_info.get("id")): return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) @@ -94,7 +94,7 @@ async def notifications_get(request: Request, encoder: Encoder) -> Response: return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode) -@route("DELETE", "api/notifications/{id}", "notification_delete") +@route("DELETE", r"api/notifications/{id:\d+}", "notification_delete") async def notifications_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response: if not (identifier := request.match_info.get("id")): return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) @@ -114,7 +114,7 @@ async def notifications_delete(request: Request, encoder: Encoder, notify: Event return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) -@route("PATCH", "api/notifications/{id}", "notification_patch") +@route("PATCH", r"api/notifications/{id:\d+}", "notification_patch") async def notifications_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response: if not (identifier := request.match_info.get("id")): return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) @@ -180,7 +180,7 @@ async def notifications_patch(request: Request, encoder: Encoder, notify: EventB ) -@route("PUT", "api/notifications/{id}", "notification_update") +@route("PUT", r"api/notifications/{id:\d+}", "notification_update") async def notifications_update(request: Request, encoder: Encoder, notify: EventBus) -> Response: if not (identifier := request.match_info.get("id")): return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) diff --git a/app/features/tasks/__init__.py b/app/features/tasks/__init__.py new file mode 100644 index 00000000..8dfaca03 --- /dev/null +++ b/app/features/tasks/__init__.py @@ -0,0 +1 @@ +"""Tasks Feature.""" diff --git a/app/features/tasks/definitions/__init__.py b/app/features/tasks/definitions/__init__.py new file mode 100644 index 00000000..9b24415c --- /dev/null +++ b/app/features/tasks/definitions/__init__.py @@ -0,0 +1 @@ +"""Tasks Definitions Feature.""" diff --git a/app/features/tasks/definitions/deps.py b/app/features/tasks/definitions/deps.py new file mode 100644 index 00000000..34133f11 --- /dev/null +++ b/app/features/tasks/definitions/deps.py @@ -0,0 +1,5 @@ +from app.features.tasks.definitions.repository import TaskDefinitionsRepository + + +def get_task_definitions_repo() -> TaskDefinitionsRepository: + return TaskDefinitionsRepository.get_instance() diff --git a/app/features/tasks/definitions/handlers/__init__.py b/app/features/tasks/definitions/handlers/__init__.py new file mode 100644 index 00000000..38c0b67e --- /dev/null +++ b/app/features/tasks/definitions/handlers/__init__.py @@ -0,0 +1 @@ +"""Handlers package for task definitions.""" diff --git a/app/library/task_handlers/_base_handler.py b/app/features/tasks/definitions/handlers/_base_handler.py similarity index 98% rename from app/library/task_handlers/_base_handler.py rename to app/features/tasks/definitions/handlers/_base_handler.py index bbf3c32f..dbe0880e 100644 --- a/app/library/task_handlers/_base_handler.py +++ b/app/features/tasks/definitions/handlers/_base_handler.py @@ -11,7 +11,7 @@ from app.library.Tasks import Task, TaskFailure, TaskResult class BaseHandler: @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: Task) -> bool: return False @staticmethod diff --git a/app/library/task_handlers/generic.py b/app/features/tasks/definitions/handlers/generic.py similarity index 50% rename from app/library/task_handlers/generic.py rename to app/features/tasks/definitions/handlers/generic.py index aac2d466..5888e920 100644 --- a/app/library/task_handlers/generic.py +++ b/app/features/tasks/definitions/handlers/generic.py @@ -8,17 +8,18 @@ import hashlib import json import logging import re -from collections.abc import Iterable, Mapping, MutableMapping -from dataclasses import dataclass, field -from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any from urllib.parse import urljoin import jmespath from parsel import Selector -from parsel.selector import SelectorList from yt_dlp.utils.networking import random_user_agent +from app.features.tasks.definitions.schemas import ( + ExtractionRule, + TaskDefinition, +) from app.library.cache import Cache from app.library.config import Config from app.library.httpx_client import async_client @@ -28,6 +29,8 @@ from app.library.Utils import fetch_info, get_archive_id from ._base_handler import BaseHandler if TYPE_CHECKING: + from pathlib import Path + import httpx from parsel.selector import SelectorList @@ -35,533 +38,6 @@ LOG: logging.Logger = logging.getLogger(__name__) CACHE: Cache = Cache() -@dataclass(slots=True) -class MatchRule: - """Represents a single URL matcher compiled to regex.""" - - source: str - """Original source pattern (regex or glob).""" - - regex: re.Pattern[str] - """Compiled regex pattern.""" - - @classmethod - def from_value(cls, value: str | Mapping[str, Any]) -> MatchRule | None: - """ - Create a MatchRule from a string or mapping. - - Args: - value (str|Mapping[str, Any]): A string (treated as glob) or a mapping with 'regex' or 'glob' keys. - - Returns: - (MatchRule|None): A MatchRule instance if successful, None otherwise. - - """ - if isinstance(value, Mapping): - pattern: str | None = value.get("regex") - glob_pattern: str | None = value.get("glob") - raw: str | None = None - - if isinstance(pattern, str) and pattern: - raw = pattern - try: - compiled: re.Pattern[str] = re.compile(pattern) - except re.error as exc: - LOG.error(f"Invalid regex pattern '{pattern}': {exc}") - return None - - return cls(source=raw, regex=compiled) - - if isinstance(glob_pattern, str) and glob_pattern: - raw = glob_pattern - compiled = re.compile(fnmatch.translate(glob_pattern)) - return cls(source=raw, regex=compiled) - - LOG.error("Matcher mapping must include 'regex' or 'glob' key with a string value.") - return None - - if not isinstance(value, str) or not value: - LOG.error(f"Matcher value must be a non-empty string, got '{value!r}'.") - return None - - # Treat plain string as glob pattern for convenience. - compiled = re.compile(fnmatch.translate(value)) - return cls(source=value, regex=compiled) - - def matches(self, url: str) -> bool: - """ - Check if the given URL matches this rule. - - Args: - url (str): The URL to check. - - Returns: - (bool): True if the URL matches, False otherwise. - - """ - return bool(self.regex.match(url)) - - -@dataclass(slots=True) -class PostFilter: - """Regex post-filter applied on extracted values.""" - - pattern: re.Pattern[str] - """Compiled regex pattern.""" - - value_key: str | None = None - """Optional group name or index to extract from the match.""" - - @classmethod - def from_mapping(cls, mapping: Mapping[str, Any]) -> PostFilter | None: - """ - Create a PostFilter from a mapping. - - Args: - mapping (Mapping[str,Any]): A mapping with 'filter' (regex pattern) and optional 'value' (group name or index) keys. - - Returns: - (PostFilter|None): A PostFilter instance if successful, None otherwise. - - """ - pattern: str | None = mapping.get("filter") - if not isinstance(pattern, str) or not pattern: - LOG.error("post_filter requires a non-empty 'filter' string.") - return None - - try: - compiled: re.Pattern[str] = re.compile(pattern) - except re.error as exc: - LOG.error(f"Invalid post_filter regex '{pattern}': {exc}") - return None - - value_key: str | None = mapping.get("value") - if value_key is not None and not isinstance(value_key, str): - LOG.error("post_filter 'value' must be a string if provided.") - return None - - return cls(pattern=compiled, value_key=value_key) - - def apply(self, candidate: str) -> str | None: - """ - Apply the post-filter to the candidate string. - - Args: - candidate (str): The string to filter. - - Returns: - (str|None): The filtered value if matched, None otherwise. - - """ - match: re.Match[str] | None = self.pattern.search(candidate) - if not match: - return None - - if self.value_key: - try: - return match.group(self.value_key) - except IndexError: - LOG.warning( - f"post_filter value index '{self.value_key}' not present in pattern {self.pattern.pattern!r}." - ) - except KeyError: - LOG.warning( - f"post_filter value key '{self.value_key}' not present in pattern {self.pattern.pattern!r}." - ) - return None - - if match.groupdict(): - # Prefer first named group when available. - key, value = next(iter(match.groupdict().items())) - if value is not None: - LOG.debug(f"post_filter using named group '{key}'.") - return value - - if match.groups(): - return match.group(1) - - return match.group(0) - - -@dataclass(slots=True) -class ExtractionRule: - """Single field extraction description.""" - - type: Literal["css", "xpath", "regex"] - """Type of extraction to perform.""" - - expression: str - """CSS selector, XPath expression or regex pattern.""" - - attribute: str | None = None - """Optional attribute to extract (e.g. 'href', 'src', 'text', etc.).""" - - post_filter: PostFilter | None = None - """Optional post-filter to apply on extracted values.""" - - -@dataclass(slots=True) -class EngineConfig: - """Engine selection to fetch the page.""" - - type: Literal["httpx", "selenium"] = "httpx" - """Engine type to use.""" - - options: dict[str, Any] = field(default_factory=dict) - """Engine-specific options.""" - - -@dataclass(slots=True) -class RequestConfig: - """HTTP request configuration.""" - - method: str = "GET" - """HTTP method to use.""" - headers: dict[str, str] = field(default_factory=dict) - """HTTP headers to include.""" - params: dict[str, Any] = field(default_factory=dict) - """Query parameters to include.""" - - data: Any | None = None - """Request body data to include.""" - json: Any | None = None - """Request body JSON data to include.""" - timeout: float | None = None - """Request timeout in seconds.""" - url: str | None = None - """Optional URL to use instead of the task URL.""" - - def normalized_method(self) -> str: - """ - Get the HTTP method in uppercase. - - Returns: - (str): The HTTP method in uppercase. - - """ - return self.method.upper() if isinstance(self.method, str) else "GET" - - -@dataclass(slots=True) -class ResponseConfig: - """Defines how to interpret the response body returned by the fetch engine.""" - - format: Literal["html", "json"] = "html" - """Body format. Defaults to HTML.""" - - -@dataclass(slots=True) -class ContainerDefinition: - """Defines a repeating element with nested field extraction.""" - - selector_type: Literal["css", "xpath", "jsonpath"] - """Type of selector to use for locating container elements.""" - - selector: str - """Selector expression for locating container elements.""" - - fields: dict[str, ExtractionRule] - """Field extraction rules relative to the container.""" - - -@dataclass(slots=True) -class TaskDefinition: - """Full task definition as loaded from disk.""" - - name: str - """Human-readable name of the task definition.""" - source: Path - """Path to the source JSON file.""" - matchers: list[MatchRule] - """List of URL matchers.""" - engine: EngineConfig - """Engine configuration.""" - request: RequestConfig - """Request configuration.""" - parsers: dict[str, ExtractionRule] - """Field extraction rules.""" - container: ContainerDefinition | None = None - """Optional container definition for repeating elements.""" - response: ResponseConfig = field(default_factory=ResponseConfig) - """Response configuration.""" - - def matches(self, url: str) -> bool: - """ - Check if the given URL matches any of the defined matchers. - - Args: - url (str): The URL to check. - - Returns: - (bool): True if any matcher matches the URL, False otherwise. - - """ - return any(rule.matches(url) for rule in self.matchers) - - -def _build_extraction_rule(field: str, raw: Mapping[str, Any], *, source: Path) -> ExtractionRule | None: - """ - Build an ExtractionRule from a raw mapping. - - Args: - field (str): The name of the field being defined. - raw (Mapping[str, Any]): The raw mapping defining the extraction rule. - source (Path): Path to the source JSON file for logging context. - - Returns: - (ExtractionRule|None): An ExtractionRule instance if successful, None otherwise. - - """ - type_value: str | None = raw.get("type") - expression: str | None = raw.get("expression") - - if not isinstance(type_value, str): - LOG.error(f"[{source.name}] Field '{field}' is missing a valid 'type'.") - return None - - if type_value not in {"css", "xpath", "regex", "jsonpath"}: - LOG.error(f"[{source.name}] Field '{field}' has unsupported type '{type_value}'.") - return None - - if not isinstance(expression, str) or not expression: - LOG.error(f"[{source.name}] Field '{field}' requires non-empty 'expression'.") - return None - - attribute: str | None = raw.get("attribute") - if attribute is not None and not isinstance(attribute, str): - LOG.error(f"[{source.name}] Field '{field}' attribute must be a string if provided.") - return None - - post_filter: PostFilter | None = None - if isinstance(raw.get("post_filter"), Mapping): - post_filter = PostFilter.from_mapping(raw["post_filter"]) - - return ExtractionRule(type=type_value, expression=expression, attribute=attribute, post_filter=post_filter) - - -def _build_matchers(raw_match: Iterable[Any], *, source: Path) -> list[MatchRule]: - """ - Build a list of MatchRule instances from raw match definitions. - - Args: - raw_match (Iterable[Any]): An iterable of raw match definitions (strings or mappings). - source (Path): Path to the source JSON file for logging context. - - Returns: - (list[MatchRule]): A list of MatchRule instances. - - """ - matchers: list[MatchRule] = [] - for value in raw_match: - rule: MatchRule | None = MatchRule.from_value(value) - if rule: - matchers.append(rule) - - if not matchers: - LOG.error(f"[{source.name}] No valid match rules found.") - - return matchers - - -def _normalize_mapping(value: Any) -> MutableMapping[str, Any]: - """ - Ensure the value is a mutable mapping. - - Args: - value (Any): The value to check. - - Returns: - (MutableMapping[str, Any]): The value if it's a mutable mapping. - - """ - if isinstance(value, MutableMapping): - return value - - msg = "Expected a mapping for parse/request/engine sections." - raise ValueError(msg) - - -def load_task_definitions(config: Config | None = None) -> list[TaskDefinition]: - """ - Load JSON task definitions from the configured tasks directory. - - Args: - config (Config|None): Optional Config instance. If None, the singleton instance is used. - - Returns: - (list[TaskDefinition]): A list of loaded TaskDefinition instances. - - """ - cfg: Config = config or Config.get_instance() - tasks_dir: Path = Path(cfg.config_path) / "tasks" - - if not tasks_dir.exists(): - return [] - - definitions: list[TaskDefinition] = [] - - for path in sorted(tasks_dir.glob("*.json")): - try: - content: str = path.read_text(encoding="utf-8") - except Exception as exc: - LOG.error(f"Failed to read task configuration '{path}': {exc}") - continue - - try: - raw = json.loads(content) - except Exception as exc: - LOG.error(f"Failed to parse JSON for '{path}': {exc}") - continue - - if not isinstance(raw, Mapping): - LOG.error(f"Task definition in '{path}' must be a JSON object.") - continue - - name: str | None = raw.get("name") - if not isinstance(name, str) or not name.strip(): - LOG.error(f"Task definition '{path}' missing a valid 'name'.") - continue - - match_value: list[str] | None = raw.get("match") - if not isinstance(match_value, Iterable) or isinstance(match_value, (str, bytes)): - LOG.error(f"[{path.name}] 'match' must be a list of patterns.") - continue - - matchers: list[MatchRule] = _build_matchers(match_value, source=path) - if not matchers: - continue - - engine_raw: Any = raw.get("engine", {}) - try: - engine_map: MutableMapping[str, Any] = _normalize_mapping(engine_raw) - except ValueError: - LOG.error(f"[{path.name}] 'engine' must be a JSON object when provided.") - continue - - engine_type: str | None = engine_map.get("type", "httpx") - if engine_type not in ("httpx", "selenium"): - LOG.error(f"[{path.name}] Unsupported engine type '{engine_type}'.") - continue - - engine_options: Any = engine_map.get("options") if isinstance(engine_map.get("options"), Mapping) else {} - engine = EngineConfig(type=engine_type, options=dict(engine_options)) - - request_raw: Any = raw.get("request", {}) - try: - request_map: MutableMapping[str, Any] = _normalize_mapping(request_raw) - except ValueError: - LOG.error(f"[{path.name}] 'request' must be a JSON object when provided.") - continue - - request = RequestConfig( - method=str(request_map.get("method", "GET")), - headers=dict(request_map.get("headers", {})) if isinstance(request_map.get("headers"), Mapping) else {}, - params=dict(request_map.get("params", {})) if isinstance(request_map.get("params"), Mapping) else {}, - data=request_map.get("data"), - json=request_map.get("json"), - timeout=float(request_map.get("timeout")) if request_map.get("timeout") is not None else None, - url=str(request_map.get("url")) if isinstance(request_map.get("url"), str) else None, - ) - - response_raw: Any = raw.get("response", {}) - response_config = ResponseConfig() - if response_raw: - if not isinstance(response_raw, Mapping): - LOG.error(f"[{path.name}] 'response' must be an object when provided.") - continue - - response_type: str = str(response_raw.get("type", "html")).lower() - if response_type not in ("html", "json"): - LOG.error(f"[{path.name}] Unsupported response type '{response_type}'.") - continue - - response_config = ResponseConfig(format=response_type) - - parse_raw: Mapping | None = raw.get("parse") - if not isinstance(parse_raw, Mapping): - LOG.error(f"[{path.name}] 'parse' must be a JSON object mapping fields to instructions.") - continue - - container_definition: ContainerDefinition | None = None - parsers: dict[str, ExtractionRule] = {} - - items_block: Mapping | None = parse_raw.get("items") - if isinstance(items_block, Mapping): - raw_fields: Mapping | None = items_block.get("fields") - if not isinstance(raw_fields, Mapping): - LOG.error(f"[{path.name}] 'items.fields' must be a mapping of field definitions.") - continue - - container_fields: dict[str, ExtractionRule] = {} - for _field, rule in raw_fields.items(): - if not isinstance(_field, str): - LOG.error(f"[{path.name}] Container field names must be strings, got {_field!r}.") - continue - - if not isinstance(rule, Mapping): - LOG.error(f"[{path.name}] Container definition for '{_field}' must be an object.") - continue - - extraction_rule: ExtractionRule | None = _build_extraction_rule(_field, rule, source=path) - if extraction_rule: - container_fields[_field] = extraction_rule - - if "link" not in container_fields: - LOG.error(f"[{path.name}] Container definition is missing required 'link' field.") - continue - - selector_value: str | None = items_block.get("selector") or items_block.get("expression") - if not isinstance(selector_value, str) or not selector_value: - LOG.error(f"[{path.name}] 'items.selector' must be a non-empty string.") - continue - - selector_type = str(items_block.get("type", "css")) - if selector_type not in ("css", "xpath", "jsonpath"): - LOG.error(f"[{path.name}] Unsupported container selector type '{selector_type}'.") - continue - - container_definition = ContainerDefinition( - selector_type=selector_type, - selector=selector_value, - fields=container_fields, - ) - - for _field, rule in parse_raw.items(): - if "items" == _field: - continue - - if not isinstance(_field, str): - LOG.error(f"[{path.name}] Parser field names must be strings, got {_field!r}.") - continue - - if not isinstance(rule, Mapping): - LOG.error(f"[{path.name}] Parser definition for '{_field}' must be an object.") - continue - - extraction_rule = _build_extraction_rule(_field, rule, source=path) - if extraction_rule: - parsers[_field] = extraction_rule - - if container_definition is None and "link" not in parsers: - LOG.error(f"[{path.name}] Missing required 'link' parser definition.") - continue - - definition = TaskDefinition( - name=name.strip(), - source=path, - matchers=matchers, - engine=engine, - request=request, - parsers=parsers, - container=container_definition, - response=response_config, - ) - - definitions.append(definition) - - return definitions - - class GenericTaskHandler(BaseHandler): """Handler that scrapes arbitrary web pages based on JSON task definitions.""" @@ -572,18 +48,7 @@ class GenericTaskHandler(BaseHandler): """Modification times of source files to detect changes.""" @classmethod - def _tasks_dir(cls) -> Path: - """ - Get the path to the tasks directory. - - Returns: - (Path): Path to the tasks directory. - - """ - return Path(Config.get_instance().config_path) / "tasks" - - @classmethod - def _refresh_definitions(cls, force: bool = False) -> None: + async def refresh_definitions(cls, force: bool = False) -> None: """ Refresh the cached task definitions if source files have changed. @@ -591,32 +56,29 @@ class GenericTaskHandler(BaseHandler): force (bool): If True, force reload even if no changes detected. """ - tasks_dir: Path = cls._tasks_dir() + if cls._definitions and not force: + return cls._definitions - if not tasks_dir.exists(): - if cls._definitions or cls._sources_mtime: - cls._definitions = [] - cls._sources_mtime = {} - return + try: + from app.features.tasks.definitions.repository import TaskDefinitionsRepository + from app.features.tasks.definitions.utils import model_to_schema - current: dict[Path, float] = {} - for path in tasks_dir.glob("*.json"): - try: - current[path] = path.stat().st_mtime - except OSError: - LOG.warning(f"Unable to stat task definition '{path}'.") + repo = TaskDefinitionsRepository.get_instance() + models = await repo.list() - if force or not cls._definitions or current != cls._sources_mtime: - cls._definitions = load_task_definitions() - cls._sources_mtime = current + definitions: list[TaskDefinition] = [] + for model in models: + td = model_to_schema(model) + definitions.append(td) + + cls._definitions = definitions + return cls._definitions + except Exception as exc: + LOG.error(f"Failed to load task definitions from database: {exc}") + return [] @classmethod - def refresh_definitions(cls, force: bool = False) -> None: - """Public helper to refresh cached task definitions.""" - cls._refresh_definitions(force=force) - - @classmethod - def _find_definition(cls, url: str) -> TaskDefinition | None: + async def _find_definition(cls, url: str) -> TaskDefinition | None: """ Find a task definition that matches the given URL. @@ -627,19 +89,30 @@ class GenericTaskHandler(BaseHandler): (TaskDefinition|None): A matching TaskDefinition if found, None otherwise. """ - cls._refresh_definitions() + await cls.refresh_definitions() for definition in cls._definitions: + if not definition.enabled: + continue + try: - if definition.matches(url): - return definition + for matcher in definition.match_url: + pattern_str = None + + if matcher.startswith("/") and matcher.endswith("/") and len(matcher) > 2: + pattern_str: str = matcher[1:-1] + else: + pattern_str = fnmatch.translate(matcher) + + if pattern_str and re.match(pattern_str, url): + return definition except Exception as exc: LOG.error(f"Error while matching definition '{definition.name}': {exc}") return None @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: Task) -> bool: """ Determine if this handler can process the given task. @@ -650,7 +123,7 @@ class GenericTaskHandler(BaseHandler): (bool): True if the handler can process the task, False otherwise. """ - definition: TaskDefinition | None = GenericTaskHandler._find_definition(task.url) + definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url) if definition: LOG.debug(f"'{task.name}': Matched generic task definition '{definition.name}'.") return True @@ -659,14 +132,14 @@ class GenericTaskHandler(BaseHandler): @staticmethod async def extract(task: Task, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004 - definition: TaskDefinition | None = GenericTaskHandler._find_definition(task.url) + definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url) if not definition: return TaskFailure(message="No generic task definition matched the provided URL.") ytdlp_opts: dict[str, Any] = task.get_ytdlp_opts().get_all() - target_url: str = definition.request.url or task.url + target_url: str = definition.definition.request.url or task.url - LOG.debug(f"{task.name!r}: Fetching '{target_url}' using engine '{definition.engine.type}'.") + LOG.debug(f"{task.name!r}: Fetching '{target_url}' using engine '{definition.definition.engine.type}'.") try: body_text, json_data = await GenericTaskHandler._fetch_content( @@ -676,10 +149,10 @@ class GenericTaskHandler(BaseHandler): LOG.exception(exc) return TaskFailure(message="Failed to fetch target URL.", error=str(exc)) - if "json" == definition.response.format and json_data is None: + if "json" == definition.definition.response.type and json_data is None: return TaskFailure(message="Expected JSON response but decoding failed.") - if "json" != definition.response.format and not body_text: + if "json" != definition.definition.response.type and not body_text: return TaskFailure(message="Received empty response body.") raw_items: list[dict[str, str]] = GenericTaskHandler._parse_items( @@ -690,9 +163,9 @@ class GenericTaskHandler(BaseHandler): def _generic_id(url): import os - import urllib + from urllib import parse - return urllib.parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0]) + return parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0]) for entry in raw_items: if not isinstance(entry, dict): @@ -701,8 +174,8 @@ class GenericTaskHandler(BaseHandler): if not (url := entry.get("link") or entry.get("url")): continue - idDict: str | None = get_archive_id(url=url) - archive_id: str | None = idDict.get("archive_id") + id_dict: dict[str, str | None] = get_archive_id(url=url) + archive_id: str | None = id_dict.get("archive_id") if not archive_id: cache_key: str = hashlib.sha256(f"{task.name}-{url}".encode()).hexdigest() if CACHE.has(cache_key): @@ -755,7 +228,7 @@ class GenericTaskHandler(BaseHandler): items=task_items, metadata={ "definition": definition.name, - "response_format": definition.response.format, + "response_format": definition.definition.response.type, }, ) @@ -770,14 +243,14 @@ class GenericTaskHandler(BaseHandler): Args: url (str): The URL to fetch. - definition (TaskDefinition): The task definition specifying the engine and request details. + definition (TaskDefinitionRuntimeSchema): The task definition specifying the engine and request details. ytdlp_opts (dict[str, Any]): yt-dlp options that may influence fetching Returns: (str|None): The fetched HTML content if successful, None otherwise. """ - if "selenium" == definition.engine.type: + if "selenium" == definition.definition.engine.type: return await GenericTaskHandler._fetch_with_selenium(url=url, definition=definition) return await GenericTaskHandler._fetch_with_httpx(url=url, definition=definition, ytdlp_opts=ytdlp_opts) @@ -793,14 +266,14 @@ class GenericTaskHandler(BaseHandler): Args: url (str): The URL to fetch. - definition (TaskDefinition): The task definition specifying the request details. + definition (TaskDefinitionRuntimeSchema): The task definition specifying the request details. ytdlp_opts (dict[str, Any]): yt-dlp options that may influence fetching Returns: (str|None): The fetched HTML content if successful, None otherwise. """ - headers: dict[str, str] = {**definition.request.headers} + headers: dict[str, str] = {**definition.definition.request.headers} client_options: dict[str, Any] = { "headers": { "User-Agent": random_user_agent(), @@ -825,20 +298,20 @@ class GenericTaskHandler(BaseHandler): if proxy := ytdlp_opts.get("proxy"): client_options["proxy"] = proxy - timeout_value: float | Any = definition.request.timeout or ytdlp_opts.get("socket_timeout", 120) + timeout_value: float | Any = definition.definition.request.timeout or ytdlp_opts.get("socket_timeout", 120) async with async_client(**client_options) as client: response: httpx.Response = await client.request( - method=definition.request.normalized_method(), + method=definition.definition.request.method.upper(), url=url, - params=definition.request.params or None, - data=definition.request.data, - json=definition.request.json, + params=definition.definition.request.params or None, + data=definition.definition.request.data, + json=definition.definition.request.json_data, timeout=timeout_value, ) response.raise_for_status() - if "json" == definition.response.format: + if "json" == definition.definition.response.type: try: json_data: dict[str, Any] = response.json() except Exception as exc: @@ -859,7 +332,7 @@ class GenericTaskHandler(BaseHandler): Args: url (str): The URL to fetch. - definition (TaskDefinition): The task definition specifying the engine options. + definition (TaskDefinitionRuntimeSchema): The task definition specifying the engine options. Returns: (str|None): The fetched HTML content if successful, None otherwise. @@ -873,15 +346,20 @@ class GenericTaskHandler(BaseHandler): from selenium.webdriver.support.ui import WebDriverWait except ImportError as exc: LOG.error(f"Selenium engine requested but selenium is not installed: {exc!s}.") - return None + return (None, None) - options_map: dict[str, Any] = definition.engine.options - command_executor: str | None = options_map.get("url", "http://localhost:4444/wd/hub") + options_map: dict[str, Any] = definition.definition.engine.options + command_executor_value = options_map.get("url") + command_executor: str = ( + str(command_executor_value) + if isinstance(command_executor_value, str) and command_executor_value + else "http://localhost:4444/wd/hub" + ) browser: str = str(options_map.get("browser", "chrome")).lower() if "chrome" != browser: LOG.error(f"Unsupported selenium browser '{browser}'. Only 'chrome' is supported.") - return None + return (None, None) arguments: list[str] | str = options_map.get("arguments", ["--headless", "--disable-gpu"]) if isinstance(arguments, str): @@ -890,8 +368,8 @@ class GenericTaskHandler(BaseHandler): wait_for: Mapping | None = ( options_map.get("wait_for") if isinstance(options_map.get("wait_for"), Mapping) else None ) - wait_timeout = float(options_map.get("wait_timeout", 15)) - page_load_timeout = float(options_map.get("page_load_timeout", 60)) + wait_timeout = float(options_map.get("wait_timeout") or 15) + page_load_timeout = float(options_map.get("page_load_timeout") or 60) def load_page() -> str | None: chrome_options = ChromeOptions() @@ -929,7 +407,7 @@ class GenericTaskHandler(BaseHandler): Parse the HTML content and extract items based on the definition. Args: - definition (TaskDefinition): The task definition specifying the parsers. + definition (TaskDefinitionRuntimeSchema): The task definition specifying the parsers. html (str): The HTML content to parse. base_url (str): The base URL to resolve relative links. json_data (Any|None): The JSON data to parse if applicable. @@ -938,12 +416,12 @@ class GenericTaskHandler(BaseHandler): (list[dict[str, str]]): A list of extracted items as dictionaries. """ - if "json" == definition.response.format: + if "json" == definition.definition.response.type: return GenericTaskHandler._parse_json_items(definition, json_data, base_url) selector = Selector(text=html) - if definition.container: + if definition.definition.parse.get("items"): return GenericTaskHandler._parse_with_container( definition=definition, selector=selector, @@ -953,7 +431,10 @@ class GenericTaskHandler(BaseHandler): extracted: dict[str, list[str]] = {} - for _field, rule in definition.parsers.items(): + for _field, rule_data in definition.definition.parse.field_items(): + if not isinstance(rule_data, dict): + continue + rule = ExtractionRule.model_validate(rule_data) values: list[str] = GenericTaskHandler._execute_rule(field=_field, selector=selector, html=html, rule=rule) extracted[_field] = values @@ -997,13 +478,16 @@ class GenericTaskHandler(BaseHandler): LOG.debug(f"Definition '{definition.name}' expects JSON but no data was parsed.") return [] - if definition.container: + if definition.definition.parse.get("items"): return GenericTaskHandler._parse_json_with_container(definition, json_data, base_url) items: list[dict[str, str]] = [] entry: dict[str, str] = {} - for _field, rule in definition.parsers.items(): + for _field, rule_data in definition.definition.parse.field_items(): + if not isinstance(rule_data, dict): + continue + rule = ExtractionRule.model_validate(rule_data) values: list[str] = GenericTaskHandler._execute_json_rule(_field, json_data, rule) if values: if "link" == _field: @@ -1023,18 +507,19 @@ class GenericTaskHandler(BaseHandler): html: str, base_url: str, ) -> list[dict[str, str]]: - container: ContainerDefinition | None = definition.container + container: dict[str, Any] | None = definition.definition.parse.get("items") if not container: return [] - if "jsonpath" == container.selector_type: - LOG.error( - f"Container selector type 'jsonpath' requires response type 'json'. Definition '{definition.name}'." - ) + container_type = container.get("type", "css") + container_selector = container.get("selector") or container.get("expression") or "" + if not container_selector: + LOG.error(f"Container missing selector/expression. Definition '{definition.name}'.") return [] + container_fields = container.get("fields", {}) selection: SelectorList[Selector] = ( - selector.css(container.selector) if "css" == container.selector_type else selector.xpath(container.selector) + selector.css(container_selector) if "css" == container_type else selector.xpath(container_selector) ) items: list[dict[str, str]] = [] @@ -1043,7 +528,8 @@ class GenericTaskHandler(BaseHandler): node_html: Any | str = node.get() or html entry: dict[str, str] = {} - for _field, rule in container.fields.items(): + for _field, rule_data in container_fields.items(): + rule = ExtractionRule.model_validate(rule_data) values: list[str] = GenericTaskHandler._execute_rule( field=_field, selector=node, @@ -1073,15 +559,19 @@ class GenericTaskHandler(BaseHandler): json_data: Any, base_url: str, ) -> list[dict[str, str]]: - container: ContainerDefinition | None = definition.container + container: dict[str, Any] | None = definition.definition.parse.get("items") if not container: return [] - if "jsonpath" != container.selector_type: + container_type = container.get("type", "css") + container_selector = container.get("selector") or container.get("expression", "") + container_fields = container.get("fields", {}) + + if "jsonpath" != container_type: LOG.error(f"JSON response requires container selector type 'jsonpath'. Definition '{definition.name}'.") return [] - nodes: Any = GenericTaskHandler._json_search(json_data, container.selector) + nodes: Any = GenericTaskHandler._json_search(json_data, container_selector) if nodes is None: return [] @@ -1093,7 +583,8 @@ class GenericTaskHandler(BaseHandler): for node in nodes: entry: dict[str, str] = {} - for _field, rule in container.fields.items(): + for _field, rule_data in container_fields.items(): + rule = ExtractionRule.model_validate(rule_data) values: list[str] = GenericTaskHandler._execute_json_rule(_field, node, rule) if not values: continue @@ -1175,7 +666,7 @@ class GenericTaskHandler(BaseHandler): field (str): The name of the field being extracted. selector (Selector): The parsel Selector for the HTML content. html (str): The raw HTML content. - rule (ExtractionRule): The extraction rule to execute. + rule (ExtractionRuleSchema): The extraction rule to execute. Returns: (list[str]): A list of extracted values. @@ -1267,7 +758,7 @@ class GenericTaskHandler(BaseHandler): if attr and attr not in {"html", "outer_html", "text", "inner_text"}: try: - attributes: dict[str, str] = sel.attrib + attributes: dict[str, str] | None = sel.attrib except AttributeError: attributes = None @@ -1281,7 +772,7 @@ class GenericTaskHandler(BaseHandler): if attr is None and "link" == field.lower(): href = None try: - attributes = sel.attrib + attributes: dict[str, str] | None = sel.attrib except AttributeError: attributes = None @@ -1307,7 +798,7 @@ class GenericTaskHandler(BaseHandler): Args: value (str|None): The extracted value to filter. - rule (ExtractionRule): The extraction rule containing the post-filter. + rule (ExtractionRuleSchema): The extraction rule containing the post-filter. Returns: (str|None): The filtered value if applicable, None otherwise. @@ -1318,6 +809,30 @@ class GenericTaskHandler(BaseHandler): cleaned: str = value.strip() if rule.post_filter: - return rule.post_filter.apply(cleaned) + # Apply post-filter inline (removed helper method) + try: + pattern = re.compile(rule.post_filter.filter) + match = pattern.search(cleaned) + if not match: + return None + + if rule.post_filter.value: + try: + return match.group(rule.post_filter.value) + except (IndexError, KeyError): + return None + + if match.groupdict(): + # Prefer first named group when available + for group_value in match.groupdict().values(): + if group_value is not None: + return group_value + + if match.groups(): + return match.group(1) + + return match.group(0) + except re.error: + return None return cleaned or None diff --git a/app/library/task_handlers/rss.py b/app/features/tasks/definitions/handlers/rss.py similarity index 99% rename from app/library/task_handlers/rss.py rename to app/features/tasks/definitions/handlers/rss.py index 6486faa3..f42dc99c 100644 --- a/app/library/task_handlers/rss.py +++ b/app/features/tasks/definitions/handlers/rss.py @@ -24,7 +24,7 @@ class RssGenericHandler(BaseHandler): ) @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: Task) -> bool: LOG.debug(f"'{task.name}': Checking if task URL is parsable RSS feed: {task.url}") return RssGenericHandler.parse(task.url) is not None diff --git a/app/library/task_handlers/tver.py b/app/features/tasks/definitions/handlers/tver.py similarity index 96% rename from app/library/task_handlers/tver.py rename to app/features/tasks/definitions/handlers/tver.py index 167a384f..f90797c6 100644 --- a/app/library/task_handlers/tver.py +++ b/app/features/tasks/definitions/handlers/tver.py @@ -10,9 +10,9 @@ LOG: logging.Logger = logging.getLogger(__name__) class TverHandler(BaseHandler): - SERIES_API = "https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/{id}" - SESSION_API = "https://platform-api.tver.jp/v2/api/platform_users/browser/create" - HEADERS = { + SERIES_API: str = "https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/{id}" + SESSION_API: str = "https://platform-api.tver.jp/v2/api/platform_users/browser/create" + HEADERS: dict[str, str] = { "x-tver-platform-type": "web", "Origin": "https://tver.jp", "Referer": "https://tver.jp/", @@ -21,7 +21,7 @@ class TverHandler(BaseHandler): RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?tver\.jp\/series\/(?Psr[a-z0-9_]+)$") @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: Task) -> bool: LOG.debug(f"Checking if task '{task.name}' is using parsable Tver series URL: {task.url}") return TverHandler.parse(task.url) is not None diff --git a/app/library/task_handlers/twitch.py b/app/features/tasks/definitions/handlers/twitch.py similarity index 97% rename from app/library/task_handlers/twitch.py rename to app/features/tasks/definitions/handlers/twitch.py index 8ba3c91b..29baec19 100644 --- a/app/library/task_handlers/twitch.py +++ b/app/features/tasks/definitions/handlers/twitch.py @@ -15,12 +15,12 @@ LOG: logging.Logger = logging.getLogger(__name__) class TwitchHandler(BaseHandler): - FEED = "https://twitchrss.appspot.com/vodonly/{handle}" + FEED: str = "https://twitchrss.appspot.com/vodonly/{handle}" RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?twitch\.tv\/(?P[a-z0-9_]{3,25})(?:\/.*)?$") @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: Task) -> bool: LOG.debug(f"Checking if task '{task.name}' is using parsable Twitch URL: {task.url}") return TwitchHandler.parse(task.url) is not None diff --git a/app/library/task_handlers/youtube.py b/app/features/tasks/definitions/handlers/youtube.py similarity index 97% rename from app/library/task_handlers/youtube.py rename to app/features/tasks/definitions/handlers/youtube.py index 4db7be5c..f29276d1 100644 --- a/app/library/task_handlers/youtube.py +++ b/app/features/tasks/definitions/handlers/youtube.py @@ -15,7 +15,7 @@ LOG: logging.Logger = logging.getLogger(__name__) class YoutubeHandler(BaseHandler): - FEED = "https://www.youtube.com/feeds/videos.xml?{type}={id}" + FEED: str = "https://www.youtube.com/feeds/videos.xml?{type}={id}" CHANNEL_REGEX: re.Pattern[str] = re.compile( r"^https?://(?:www\.)?youtube\.com/(?:channel/(?PUC[0-9A-Za-z_-]{22})|)/?$" @@ -26,7 +26,7 @@ class YoutubeHandler(BaseHandler): ) @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: Task) -> bool: LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}") return YoutubeHandler.parse(task.url) is not None diff --git a/app/features/tasks/definitions/migration.py b/app/features/tasks/definitions/migration.py new file mode 100644 index 00000000..74d519c9 --- /dev/null +++ b/app/features/tasks/definitions/migration.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from app.features.core.migration import Migration as FeatureMigration +from app.library.config import Config + +if TYPE_CHECKING: + from app.features.tasks.definitions.repository import TaskDefinitionsRepository + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Migration(FeatureMigration): + name: str = "task_definitions" + + def __init__(self, repo: TaskDefinitionsRepository, config: Config | None = None) -> None: + self._config: Config = config or Config.get_instance() + super().__init__(config=self._config) + self._repo: TaskDefinitionsRepository = repo + self._source_dir: Path = Path(self._config.config_path) / "tasks" + + async def should_run(self) -> bool: + if not self._source_dir.exists(): + return False + + return any(self._source_dir.glob("*.json")) + + async def migrate(self) -> None: + if await self._repo.count() > 0: + LOG.warning("Task definitions already exist in the database; skipping migration.") + await self._archive_sources() + return + + inserted = 0 + seen_names: dict[str, int] = {} + + for path in sorted(self._source_dir.glob("*.json")): + normalized = await self._normalize(path, seen_names) + if not normalized: + await self._move_file(path) + continue + + try: + await self._repo.create(normalized) + inserted += 1 + except Exception as exc: + LOG.exception("Failed to insert task definition '%s': %s", normalized.get("name"), exc) + finally: + await self._move_file(path) + + LOG.info("Migrated %s task definition(s) from %s.", inserted, self._source_dir) + + async def _archive_sources(self) -> None: + for path in self._source_dir.glob("*.json"): + await self._move_file(path) + + async def _normalize(self, path: Path, seen_names: dict[str, int]) -> dict[str, Any] | None: + try: + content = path.read_text(encoding="utf-8") + except Exception as exc: + LOG.error("Failed to read task definition '%s': %s", path, exc) + return None + + try: + payload = json.loads(content) + except Exception as exc: + LOG.error("Failed to parse JSON for '%s': %s", path, exc) + return None + + if not isinstance(payload, dict): + LOG.error("Task definition in '%s' must be a JSON object.", path) + return None + + if "match" in payload and "match_url" not in payload: + payload["match_url"] = payload.pop("match") + + # Normalize match_url from old object format to new string format + if "match_url" in payload and isinstance(payload["match_url"], list): + normalized_match: list[str] = [] + for item in payload["match_url"]: + if isinstance(item, str): + normalized_match.append(item) + elif isinstance(item, dict): + if "regex" in item and isinstance(item["regex"], str): + # Convert {regex: "pattern"} to /pattern/ + normalized_match.append(f"/{item['regex']}/") + elif "glob" in item and isinstance(item["glob"], str): + # Convert {glob: "pattern"} to pattern + normalized_match.append(item["glob"]) + payload["match_url"] = normalized_match + + # Rename request.json to request.json_data + if "request" in payload and isinstance(payload["request"], dict) and "json" in payload["request"]: + payload["request"]["json_data"] = payload["request"].pop("json") + + if "definition" not in payload: + definition_fields = {} + for field in ["parse", "engine", "request", "response"]: + if field in payload: + definition_fields[field] = payload.pop(field) + + if definition_fields: + payload["definition"] = definition_fields + # Also handle nested definition.request.json + elif isinstance(payload["definition"], dict): + if ( + "request" in payload["definition"] + and isinstance(payload["definition"]["request"], dict) + and "json" in payload["definition"]["request"] + ): + payload["definition"]["request"]["json_data"] = payload["definition"]["request"].pop("json") + + name_value = payload.get("name") + if not isinstance(name_value, str) or not name_value.strip(): + LOG.error("Task definition in '%s' missing a valid name.", path) + return None + + name = self._unique_name(name_value.strip(), seen_names) + payload["name"] = name + + # Repository will handle validation and field extraction + return payload diff --git a/app/features/tasks/definitions/models.py b/app/features/tasks/definitions/models.py new file mode 100644 index 00000000..def000fd --- /dev/null +++ b/app/features/tasks/definitions/models.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from datetime import datetime # noqa: TC003 + +from sqlalchemy import JSON, Boolean, Index, Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.features.core.models import Base, UTCDateTime, utcnow + + +class TaskDefinitionModel(Base): + __tablename__: str = "task_definitions" + __table_args__: tuple[Index, ...] = ( + Index("ix_task_definitions_name", "name"), + Index("ix_task_definitions_priority", "priority"), + Index("ix_task_definitions_match_url", "match_url"), + Index("ix_task_definitions_enabled", "enabled"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + match_url: Mapped[list] = mapped_column(JSON, nullable=False) + definition: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False) + updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False) diff --git a/app/features/tasks/definitions/repository.py b/app/features/tasks/definitions/repository.py new file mode 100644 index 00000000..80dbcf4b --- /dev/null +++ b/app/features/tasks/definitions/repository.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, or_, select + +from app.features.core.deps import get_session +from app.features.core.schemas import CEFeature, ConfigEvent +from app.features.tasks.definitions.migration import Migration +from app.features.tasks.definitions.models import TaskDefinitionModel +from app.library.Events import Event, EventBus, Events +from app.library.Services import Services +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from sqlalchemy.engine.result import Result + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Select + +LOG: logging.Logger = logging.getLogger(__name__) + + +class TaskDefinitionsRepository(metaclass=Singleton): + def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + self._migrated = False + self.session: AsyncGenerator[AsyncSession] = session or get_session + + async def run_migrations(self) -> None: + if self._migrated: + return + + self._migrated = True + await Migration(repo=self).run() + + @staticmethod + def get_instance() -> TaskDefinitionsRepository: + return TaskDefinitionsRepository() + + def attach(self, _: Any) -> None: + async def handle_event(_, __): + await self.run_migrations() + + async def handler(e: Event, __): + from app.features.tasks.definitions.handlers.generic import GenericTaskHandler + + if isinstance(e.data, ConfigEvent) and CEFeature.TASKS_DEFINITIONS == e.data.feature: + LOG.debug("Refreshing task definitions due to configuration update.") + await GenericTaskHandler.refresh_definitions(force=True) + + Services.get_instance().add(__class__.__name__, self) + EventBus.get_instance().subscribe( + Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations" + ).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions") + + async def list(self) -> list[TaskDefinitionModel]: + async with self.session() as session: + result: Result[tuple[TaskDefinitionModel]] = await session.execute( + select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc()) + ) + return list(result.scalars().all()) + + async def list_paginated(self, page: int, per_page: int) -> tuple[list[TaskDefinitionModel], int, int, int]: + async with self.session() as session: + total: int = await self.count() + total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1 + + if page > total_pages and total > 0: + page = total_pages + + query: Select[tuple[TaskDefinitionModel]] = ( + select(TaskDefinitionModel) + .order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc()) + .limit(per_page) + .offset((page - 1) * per_page) + ) + result: Result[tuple[TaskDefinitionModel]] = await session.execute(query) + return list(result.scalars().all()), total, page, total_pages + + async def count(self) -> int: + async with self.session() as session: + result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(TaskDefinitionModel)) + return int(result.scalar_one()) + + async def get(self, identifier: int | str) -> TaskDefinitionModel | None: + async with self.session() as session: + if not identifier: + return None + + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskDefinitionModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskDefinitionModel.id == int(identifier), TaskDefinitionModel.name == identifier) + else: + clause = TaskDefinitionModel.name == identifier + + result: Result[tuple[TaskDefinitionModel]] = await session.execute( + select(TaskDefinitionModel).where(clause).limit(1) + ) + return result.scalar_one_or_none() + + async def get_by_name(self, name: str, exclude_id: int | None = None) -> TaskDefinitionModel | None: + async with self.session() as session: + query: Select[tuple[TaskDefinitionModel]] = select(TaskDefinitionModel).where( + TaskDefinitionModel.name == name + ) + if exclude_id is not None: + query = query.where(TaskDefinitionModel.id != exclude_id) + + result: Result[tuple[TaskDefinitionModel]] = await session.execute(query.limit(1)) + return result.scalar_one_or_none() + + async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel: + async with self.session() as session: + model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload + if model.id is not None: + model.id = None + + if await self.get_by_name(name=model.name) is not None: + msg: str = f"Task definition with name '{model.name}' already exists." + raise ValueError(msg) + + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def update(self, identifier: int | str, payload: dict[str, Any]) -> TaskDefinitionModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskDefinitionModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskDefinitionModel.id == int(identifier), TaskDefinitionModel.name == identifier) + else: + clause = TaskDefinitionModel.name == identifier + + result: Result[tuple[TaskDefinitionModel]] = await session.execute( + select(TaskDefinitionModel).where(clause).limit(1) + ) + model: TaskDefinitionModel | None = result.scalar_one_or_none() + + if not model: + msg: str = f"Task definition '{identifier}' not found." + raise KeyError(msg) + + payload.pop("id", None) + payload.pop("created_at", None) + payload.pop("updated_at", None) + + if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None: + msg = f"Task definition with name '{payload['name']}' already exists." + raise ValueError(msg) + + for key, value in payload.items(): + if hasattr(model, key): + setattr(model, key, value) + + await session.commit() + await session.refresh(model) + return model + + async def delete(self, identifier: int | str) -> TaskDefinitionModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskDefinitionModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskDefinitionModel.id == int(identifier), TaskDefinitionModel.name == identifier) + else: + clause = TaskDefinitionModel.name == identifier + + result: Result[tuple[TaskDefinitionModel]] = await session.execute( + select(TaskDefinitionModel).where(clause).limit(1) + ) + + if not (model := result.scalar_one_or_none()): + msg: str = f"Task definition '{identifier}' not found." + raise KeyError(msg) + + await session.delete(model) + await session.commit() + return model diff --git a/app/features/tasks/definitions/router.py b/app/features/tasks/definitions/router.py new file mode 100644 index 00000000..56ee5054 --- /dev/null +++ b/app/features/tasks/definitions/router.py @@ -0,0 +1,235 @@ +import logging +from typing import Any + +from aiohttp import web +from aiohttp.web import Request, Response +from pydantic import ValidationError + +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination +from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination +from app.features.tasks.definitions.repository import TaskDefinitionsRepository as Repo +from app.features.tasks.definitions.schemas import ( + TaskDefinition, + TaskDefinitionList, + TaskDefinitionPatch, +) +from app.features.tasks.definitions.utils import model_to_schema, schema_to_payload +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.router import route + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/tasks/definitions/", "task_definitions") +async def task_definitions_list(request: Request, encoder: Encoder, repo: Repo) -> Response: + page, per_page = normalize_pagination(request) + models, total, current_page, total_pages = await repo.list_paginated(page, per_page) + + include: str | None = request.query.get("include") + summary: bool = "definition" != include + + return web.json_response( + data=TaskDefinitionList( + items=[model_to_schema(model, summary=summary) for model in models], + pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)), + ), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("GET", r"api/tasks/definitions/{id:\d+}", "task_definitions_get") +async def task_definitions_get(request: Request, encoder: Encoder, repo: Repo) -> Response: + identifier: str = request.match_info.get("id", "").strip() + if not identifier: + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + if not (model := await repo.get(identifier)): + return web.json_response( + data={"error": f"Task definition '{identifier}' not found."}, + status=web.HTTPNotFound.status_code, + ) + + definition = model_to_schema(model) + return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/tasks/definitions/", "task_definitions_create") +async def task_definitions_create(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response: + try: + payload: Any = await request.json() + except Exception: + return web.json_response( + data={"error": "Invalid JSON in request body."}, + status=web.HTTPBadRequest.status_code, + ) + + if not isinstance(payload, dict): + return web.json_response( + data={"error": "Invalid request body; expected JSON object."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition_input = TaskDefinition.model_validate(payload) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task definition.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + repo_payload = schema_to_payload(definition_input) + model = await repo.create(repo_payload) + definition = model_to_schema(model) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.CREATE, data=definition.model_dump()), + ) + return web.json_response(data=definition, status=web.HTTPCreated.status_code, dumps=encoder.encode) + except ValueError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code) + except Exception as exc: + LOG.exception(exc) + return web.json_response( + data={"error": "Failed to create task definition."}, + status=web.HTTPInternalServerError.status_code, + ) + + +@route("PUT", r"api/tasks/definitions/{id:\d+}", "task_definitions_update") +async def task_definitions_update(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response: + if not (identifier := request.match_info.get("id", "").strip()): + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + payload: dict | None = await request.json() + except Exception: + return web.json_response( + data={"error": "Invalid JSON in request body."}, + status=web.HTTPBadRequest.status_code, + ) + + if not isinstance(payload, dict): + return web.json_response( + data={"error": "Invalid request body; expected JSON object."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition_input: TaskDefinition = TaskDefinition.model_validate(payload) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task definition.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition: TaskDefinition = model_to_schema(await repo.update(identifier, schema_to_payload(definition_input))) + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.UPDATE, data=definition.model_dump()), + ) + return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode) + except KeyError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code) + except ValueError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code) + except Exception as exc: + LOG.exception(exc) + return web.json_response( + data={"error": "Failed to update task definition."}, + status=web.HTTPInternalServerError.status_code, + ) + + +@route("PATCH", r"api/tasks/definitions/{id:\d+}", "task_definitions_patch") +async def task_definitions_patch(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response: + if not (identifier := request.match_info.get("id", "").strip()): + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + if not await repo.get(identifier): + return web.json_response( + data={"error": f"Task definition '{identifier}' not found."}, + status=web.HTTPNotFound.status_code, + ) + + try: + payload: dict | None = await request.json() + except Exception: + return web.json_response( + data={"error": "Invalid JSON in request body."}, + status=web.HTTPBadRequest.status_code, + ) + + if not isinstance(payload, dict): + return web.json_response( + data={"error": "Invalid request body; expected JSON object."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + patch_input: TaskDefinitionPatch = TaskDefinitionPatch.model_validate(payload) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task definition patch.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition: TaskDefinition = model_to_schema( + await repo.update(identifier, patch_input.model_dump(exclude_unset=True)) + ) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.UPDATE, data=definition.model_dump()), + ) + return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode) + except KeyError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code) + except ValueError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code) + except Exception as exc: + LOG.exception(exc) + return web.json_response( + data={"error": "Failed to patch task definition."}, + status=web.HTTPInternalServerError.status_code, + ) + + +@route("DELETE", r"api/tasks/definitions/{id:\d+}", "task_definitions_delete") +async def task_definitions_delete(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response: + if not (identifier := request.match_info.get("id", "").strip()): + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition: TaskDefinition = model_to_schema(await repo.delete(identifier)) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.DELETE, data=definition.model_dump()), + ) + return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode) + except KeyError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code) + except Exception as exc: + LOG.exception(exc) + return web.json_response( + data={"error": "Failed to delete task definition."}, + status=web.HTTPInternalServerError.status_code, + ) diff --git a/app/features/tasks/definitions/schemas.py b/app/features/tasks/definitions/schemas.py new file mode 100644 index 00000000..19412e20 --- /dev/null +++ b/app/features/tasks/definitions/schemas.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import re +from datetime import datetime # noqa: TC003 +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from app.features.core.schemas import Pagination +from app.features.core.utils import parse_int + + +class PostFilter(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + filter: str = Field(min_length=1) + value: str | None = None + + @field_validator("filter") + @classmethod + def _validate_filter(cls, value: str) -> str: + try: + re.compile(value) + except re.error as exc: + msg: str = f"Invalid post_filter regex pattern: {exc}" + raise ValueError(msg) from exc + return value + + +class ExtractionRule(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + type: Literal["css", "xpath", "regex", "jsonpath"] + expression: str = Field(min_length=1) + attribute: str | None = None + post_filter: PostFilter | None = None + + def __getitem__(self, key: str) -> Any: + """Support bracket notation access.""" + if not hasattr(self, key): + raise KeyError(key) + return getattr(self, key) + + +class ParseItems(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + type: Literal["css", "xpath", "jsonpath"] = "css" + selector: str | None = Field(None, min_length=1) + expression: str | None = Field(None, min_length=1) + fields: dict[str, ExtractionRule] + + def get(self, key: str, default: Any = None) -> Any: + """Get a field value by key, supporting dict-like access.""" + return getattr(self, key, default) + + def __getitem__(self, key: str) -> Any: + """Support bracket notation access.""" + if not hasattr(self, key): + raise KeyError(key) + return getattr(self, key) + + @model_validator(mode="after") + def _validate_items(self) -> ParseItems: + if not self.selector and not self.expression: + msg = "Either 'selector' or 'expression' must be provided." + raise ValueError(msg) + if not self.selector: + self.selector = self.expression + if "link" not in self.fields: + msg = "Container 'fields' must include a 'link' field." + raise ValueError(msg) + return self + + +class Parse(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, extra="allow") + items: ParseItems | None = None + + def get(self, key: str, default: Any = None) -> Any: + """Get a field value by key, supporting dict-like access.""" + return getattr(self, key, default) + + def field_items(self) -> list[tuple[str, Any]]: + """Return field items like a dict, excluding private fields and 'items'.""" + data: dict[str, Any] = self.model_dump() + return [(k, v) for k, v in data.items() if k not in ("items",)] + + def __getitem__(self, key: str) -> Any: + """Support bracket notation access.""" + if not hasattr(self, key): + raise KeyError(key) + return getattr(self, key) + + def __contains__(self, key: str) -> bool: + """Support 'in' operator.""" + return hasattr(self, key) and not key.startswith("_") + + @model_validator(mode="before") + @classmethod + def _validate_parse(cls, value: Any) -> Any: + """Validate that we have either items or direct parsers with link.""" + if not isinstance(value, dict): + msg: str = "Parse must be a dict" + raise ValueError(msg) + + has_items: bool = "items" in value and value["items"] is not None + direct_parsers: dict[str, Any] = { + k: v for k, v in value.items() if k not in ("items",) and not k.startswith("_") + } + has_direct_parsers: bool = len(direct_parsers) > 0 + has_link_parser: bool = "link" in direct_parsers + + if not has_items and not has_direct_parsers: + msg: str = "Field 'parse' must contain either 'items' or direct parsers." + raise ValueError(msg) + + if not has_items and not has_link_parser: + msg: str = "Missing required 'link' parser definition." + raise ValueError(msg) + + for field_name, field_value in direct_parsers.items(): + if not isinstance(field_value, dict): + msg: str = f"Parse field '{field_name}' must be an object." + raise ValueError(msg) + + return value + + +class EngineConfig(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + type: Literal["httpx", "selenium"] = "httpx" + options: dict[str, Any] = Field(default_factory=dict) + + +class RequestConfig(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, protected_namespaces=()) + method: str = "GET" + headers: dict[str, str] = Field(default_factory=dict) + params: dict[str, Any] = Field(default_factory=dict) + data: dict[str, Any] | None = None + json_data: dict[str, Any] | None = None + timeout: float | None = None + url: str | None = None + + +class ResponseConfig(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + type: Literal["html", "json"] = "html" + + +class Definition(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + parse: Parse + engine: EngineConfig = Field(default_factory=EngineConfig) + request: RequestConfig = Field(default_factory=RequestConfig) + response: ResponseConfig = Field(default_factory=ResponseConfig) + + +class TaskDefinitionSummary(BaseModel): + model_config = ConfigDict(from_attributes=True, str_strip_whitespace=True) + id: int | None = None + name: str = Field(min_length=1) + priority: int = Field(default=0, ge=0) + match_url: list[str] = Field(min_length=1) + enabled: bool = Field(default=True) + created_at: datetime | None = None + updated_at: datetime | None = None + + +class TaskDefinition(TaskDefinitionSummary): + definition: Definition + + @field_validator("priority", mode="before") + @classmethod + def _normalize_priority(cls, value: Any) -> int: + if value is None: + return 0 + return parse_int(value, field="Priority", minimum=0) + + @field_validator("match_url", mode="before") + @classmethod + def _validate_match_url(cls, value: Any) -> list[str]: + """Validate that match_url is a list of strings and validate regex patterns.""" + if not isinstance(value, list): + msg = "match_url must be a list" + raise ValueError(msg) + + validated: list[str] = [] + for item in value: + if not isinstance(item, str): + msg: str = f"match_url items must be strings, got {type(item).__name__}" + raise ValueError(msg) + + item: str = item.strip() + if not item: + msg = "match_url items cannot be empty" + raise ValueError(msg) + + if item.startswith("/") and item.endswith("/") and len(item) > 2: + pattern = item[1:-1] + try: + re.compile(pattern) + except re.error as exc: + msg = f"Invalid regex pattern '{pattern}': {exc}" + raise ValueError(msg) from exc + + validated.append(item) + + return validated + + +class TaskDefinitionPatch(TaskDefinition): + model_config = ConfigDict(str_strip_whitespace=True) + name: str | None = None + priority: int | None = None + match_url: list[str] | None = None + definition: Definition | None = None + enabled: bool | None = None + + +class TaskDefinitionList(BaseModel): + model_config = ConfigDict(from_attributes=True) + items: list[TaskDefinitionSummary | TaskDefinition] = Field(default_factory=list) + pagination: Pagination diff --git a/app/features/tasks/definitions/tests/test_generic_task_handler.py b/app/features/tasks/definitions/tests/test_generic_task_handler.py new file mode 100644 index 00000000..31fd4161 --- /dev/null +++ b/app/features/tasks/definitions/tests/test_generic_task_handler.py @@ -0,0 +1,395 @@ +from datetime import datetime +from unittest.mock import patch + +import pytest + +from app.features.tasks.definitions.handlers.generic import GenericTaskHandler +from app.features.tasks.definitions.schemas import ( + EngineConfig, + RequestConfig, + ResponseConfig, + TaskDefinition, + Definition, +) +from app.library.Tasks import Task, TaskFailure, TaskResult + + +@pytest.fixture(autouse=True) +def reset_generic_handler(monkeypatch): + monkeypatch.setattr(GenericTaskHandler, "_definitions", []) + monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {}) + + +def test_build_task_definition_parses_valid_payload(): + definition = TaskDefinition( + id=1, + name="example", + priority=0, + match_url=["https://example.com/articles/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "link": {"type": "css", "expression": ".article a.link::attr(href)"}, + "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(), + ), + ) + + assert definition is not None, "TaskDefinition should be created" + assert "example" == definition.name, "Name should match" + assert "https://example.com/articles/*" in definition.match_url, "Match URL should be in list" + assert "link" in definition.definition.parse, "Parse should contain link field" + assert ".article a.link::attr(href)" == definition.definition.parse["link"]["expression"], ( + "Link expression should match" + ) + + +def test_build_task_definition_handles_container(): + definition = TaskDefinition( + id=2, + name="container", + priority=0, + match_url=["https://example.com/cards"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "selector": ".cards .card", + "fields": { + "link": {"type": "css", "expression": ".card-header a", "attribute": "href"}, + "title": {"type": "css", "expression": ".card-header a", "attribute": "text"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(), + ), + ) + + assert definition is not None, "TaskDefinition should be created" + assert "items" in definition.definition.parse, "Parse should contain items container" + assert ".cards .card" == definition.definition.parse["items"]["selector"], "Items selector should match" + assert "link" in definition.definition.parse["items"]["fields"], "Items fields should contain link" + + +def test_build_task_definition_handles_json(): + definition = TaskDefinition( + id=3, + name="json-def", + priority=0, + match_url=["https://example.com/api"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "jsonpath", + "selector": "items", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(type="json"), + ), + ) + + assert definition is not None, "TaskDefinition should be created" + assert "json" == definition.definition.response.type, "Response type should be json" + assert "items" in definition.definition.parse, "Parse should contain items container" + assert "jsonpath" == definition.definition.parse["items"]["type"], "Items type should be jsonpath" + assert "jsonpath" == definition.definition.parse["items"]["fields"]["link"]["type"], ( + "Link field type should be jsonpath" + ) + + +def test_parse_items_extracts_values(): + definition = TaskDefinition( + id=4, + name="example", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None}, + "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, + "id": {"type": "css", "expression": ".article", "attribute": "data-id"}, + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(), + ), + ) + + html = """ +
+ First + First Title +
+
+ Second + Second Title +
+ """ + + items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/") + + assert 2 == len(items), "Should extract 2 items" + assert "https://example.com/article-101" == items[0]["link"], "First item link should be absolute URL" + assert "First Title" == items[0]["title"], "First item title should match" + assert "101" == items[0]["id"], "First item id should match" + assert "https://example.com/article-102" == items[1]["link"], "Second item link should match" + + +def test_parse_items_handles_nested_card_layout(): + definition = TaskDefinition( + id=5, + name="nested", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "css", + "selector": ".columns .card", + "fields": { + "link": { + "type": "css", + "expression": ".card-header a[href]", + "attribute": "href", + }, + "title": { + "type": "css", + "expression": ".card-header a[href]", + "attribute": "text", + }, + "poet": { + "type": "css", + "expression": "footer .card-footer-item:first-child a", + "attribute": "text", + }, + "category": { + "type": "css", + "expression": "footer .card-footer-item:nth-child(2) a", + "attribute": "text", + }, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(), + ), + ) + + html = """ +
+
+
+
+

+ First Poem +

+
+ +
+
+
+
+
+

+ Second Poem +

+
+ +
+
+
+ """ + + items = GenericTaskHandler._parse_items(definition, html, "https://example.com") + + assert 2 == len(items), "Should extract 2 items" + assert "https://example.com/poems/view/111" == items[0]["link"], "First item link should match" + assert "First Poem" == items[0]["title"], "First item title should match" + assert "Poet Alpha" == items[0]["poet"], "First item poet should match" + assert "Category One" == items[0]["category"], "First item category should match" + + assert "https://example.com/poems/view/222" == items[1]["link"], "Second item link should match" + assert "Second Poem" == items[1]["title"], "Second item title should match" + assert "Poet Beta" == items[1]["poet"], "Second item poet should match" + assert "category" not in items[1], "Second item should not have category" + + +def test_parse_items_handles_json_container(): + definition = TaskDefinition( + id=6, + name="json", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "jsonpath", + "selector": "entries", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + "id": {"type": "jsonpath", "expression": "id"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(type="json"), + ), + ) + + payload = { + "entries": [ + {"url": "/video/1", "title": "First", "id": 1}, + {"url": "https://example.com/video/2", "title": "Second", "id": 2}, + {"title": "Missing Link", "id": 3}, + ] + } + + items = GenericTaskHandler._parse_items( + definition=definition, + html="", + base_url="https://example.com", + json_data=payload, + ) + + assert 2 == len(items), "Should extract 2 items (third missing link)" + assert "https://example.com/video/1" == items[0]["link"], "First item link should be absolute" + assert "First" == items[0]["title"], "First item title should match" + assert "1" == items[0]["id"], "First item id should match" + + assert "https://example.com/video/2" == items[1]["link"], "Second item link should match" + assert "Second" == items[1]["title"], "Second item title should match" + assert "2" == items[1]["id"], "Second item id should match" + + +@pytest.mark.asyncio +async def test_generic_task_handler_inspect(monkeypatch): + definition = TaskDefinition( + id=7, + name="json-inspect", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "jsonpath", + "selector": "items", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(type="json"), + ), + ) + + async def fake_find_definition(cls, url): # noqa: ARG001 + return definition + + monkeypatch.setattr( + GenericTaskHandler, + "_find_definition", + classmethod(fake_find_definition), + ) + + async def fake_fetch_content(url, definition, ytdlp_opts): # noqa: ARG001 + return "", {"items": [{"url": "/video/1", "title": "First"}]} + + monkeypatch.setattr(GenericTaskHandler, "_fetch_content", staticmethod(fake_fetch_content)) + + # Mock fetch_info to return valid info with required fields for archive ID generation + async def fake_fetch_info(config, url, **kwargs): # noqa: ARG001 + return {"id": "test_video_1", "extractor_key": "Example"} + + with patch("app.features.tasks.definitions.handlers.generic.fetch_info", side_effect=fake_fetch_info): + task = Task(id="inspect", name="Inspect", url="https://example.com/api") + result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task) + + assert isinstance(result, TaskResult), "Result should be TaskResult" + assert 1 == len(result.items), "Should have 1 item" + item = result.items[0] + assert "https://example.com/video/1" == item.url, "Item URL should match" + assert "First" == item.title, "Item title should match" + + +def test_parse_items_handles_json_top_level_list(): + definition = TaskDefinition( + id=8, + name="json-list", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "jsonpath", + "selector": "[]", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(type="json"), + ), + ) + + payload = [ + {"url": "/video/1", "title": "First"}, + {"url": "/video/2", "title": "Second"}, + ] + + items = GenericTaskHandler._parse_items( + definition=definition, + html="", + base_url="https://example.com", + json_data=payload, + ) + + assert 2 == len(items), "Should extract 2 items" + assert "https://example.com/video/1" == items[0]["link"], "First item link should match" + assert "First" == items[0]["title"], "First item title should match" + assert "https://example.com/video/2" == items[1]["link"], "Second item link should match" + assert "Second" == items[1]["title"], "Second item title should match" diff --git a/app/tests/test_rss_handler.py b/app/features/tasks/definitions/tests/test_rss_handler.py similarity index 97% rename from app/tests/test_rss_handler.py rename to app/features/tasks/definitions/tests/test_rss_handler.py index b8a5cb0a..be241892 100644 --- a/app/tests/test_rss_handler.py +++ b/app/features/tasks/definitions/tests/test_rss_handler.py @@ -1,6 +1,6 @@ import pytest -from app.library.task_handlers.rss import RssGenericHandler +from app.features.tasks.definitions.handlers.rss import RssGenericHandler from app.library.Tasks import Task, TaskResult @@ -155,7 +155,7 @@ class TestRssHandlerExtraction: preset="default", ) - assert RssGenericHandler.can_handle(task) is True + assert await RssGenericHandler.can_handle(task) is True non_feed_task = Task( id="test_youtube", @@ -164,7 +164,7 @@ class TestRssHandlerExtraction: preset="default", ) - assert RssGenericHandler.can_handle(non_feed_task) is False + assert await RssGenericHandler.can_handle(non_feed_task) is False class TestRssHandlerEdgeCases: diff --git a/app/features/tasks/definitions/tests/test_task_definitions.py b/app/features/tasks/definitions/tests/test_task_definitions.py new file mode 100644 index 00000000..388bd3b4 --- /dev/null +++ b/app/features/tasks/definitions/tests/test_task_definitions.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import json +from collections.abc import AsyncGenerator +from unittest.mock import AsyncMock, MagicMock + +import pytest +import pytest_asyncio +from aiohttp import web +from aiohttp.web import Request + +from app.features.tasks.definitions.repository import TaskDefinitionsRepository +from app.features.tasks.definitions.router import ( + task_definitions_create, + task_definitions_delete, + task_definitions_get, + task_definitions_list, + task_definitions_patch, + task_definitions_update, +) +from app.library.encoder import Encoder +from app.library.sqlite_store import SqliteStore +from app.main import EventBus + + +def _sample_definition(name: str = "example", *, priority: int = 0) -> dict: + """Returns a properly structured task definition payload for the repository.""" + return { + "name": name, + "match_url": ["https://example.com/*"], + "priority": priority, + "definition": { + "parse": { + "link": { + "type": "css", + "expression": "a", + "attribute": "href", + } + }, + "engine": {"type": "httpx"}, + "request": {}, + "response": {"type": "html"}, + }, + } + + +@pytest_asyncio.fixture +async def repo() -> AsyncGenerator[TaskDefinitionsRepository, None]: + TaskDefinitionsRepository._reset_singleton() + SqliteStore._reset_singleton() + + store = SqliteStore(db_path=":memory:") + await store.get_connection() + + repository = TaskDefinitionsRepository.get_instance() + + yield repository + + if store._conn: + await store._conn.close() + if store._engine: + await store._engine.dispose() + + TaskDefinitionsRepository._reset_singleton() + SqliteStore._reset_singleton() + + +class TestTaskDefinitionsRepository: + @pytest.mark.asyncio + async def test_create_and_list(self, repo: TaskDefinitionsRepository) -> None: + await repo.create(_sample_definition("Alpha", priority=2)) + await repo.create(_sample_definition("Beta", priority=1)) + + items = await repo.list() + assert len(items) == 2, "Should return two task definitions" + assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name" + + @pytest.mark.asyncio + async def test_create_duplicate_name_raises(self, repo: TaskDefinitionsRepository) -> None: + payload = _sample_definition("Dup") + await repo.create(payload) + + with pytest.raises(ValueError, match="already exists"): + await repo.create(payload) + + with pytest.raises(ValueError, match="already exists"): + await repo.create(payload) + + @pytest.mark.asyncio + async def test_update_missing_raises(self, repo: TaskDefinitionsRepository) -> None: + with pytest.raises(KeyError, match="not found"): + await repo.update(999, {"name": "Missing"}) + + @pytest.mark.asyncio + async def test_list_paginated(self, repo: TaskDefinitionsRepository) -> None: + for idx in range(5): + await repo.create(_sample_definition(f"Item {idx}", priority=idx)) + + items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2) + assert len(items) == 2, "Should return two items per page" + assert total == 5, "Should report total count of 5" + assert page == 1, "Should return requested page" + assert total_pages == 3, "Should compute total pages" + + +@pytest.mark.asyncio +class TestTaskDefinitionRoutes: + async def test_list_definitions(self, repo: TaskDefinitionsRepository) -> None: + await repo.create(_sample_definition("Sample")) + request = MagicMock(spec=Request) + request.query = {} + + response = await task_definitions_list(request, Encoder(), repo) + payload = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code, "Should return 200 for list" + assert payload["items"][0]["name"] == "Sample", "Should include created definition" + + async def test_get_definition_not_found(self, repo: TaskDefinitionsRepository) -> None: + request = MagicMock(spec=Request) + request.match_info = {"id": "999"} + + response = await task_definitions_get(request, Encoder(), repo) + payload = json.loads(response.text) + + assert response.status == web.HTTPNotFound.status_code, "Should return 404 for missing definition" + assert "error" in payload, "Should include error payload" + + async def test_create_definition_success(self, repo: TaskDefinitionsRepository) -> None: + request = MagicMock(spec=Request) + request.json = AsyncMock(return_value=_sample_definition("New", priority=3)) + + response = await task_definitions_create(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPCreated.status_code, "Should create task definition" + assert body["name"] == "New", "Should return created name" + assert body["priority"] == 3, "Should return created priority" + + async def test_update_definition_success(self, repo: TaskDefinitionsRepository) -> None: + created = await repo.create(_sample_definition("Original", priority=0)) + + request = MagicMock(spec=Request) + request.match_info = {"id": str(created.id)} + request.json = AsyncMock(return_value=_sample_definition("Updated", priority=4)) + + response = await task_definitions_update(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code, "Should update task definition" + assert body["name"] == "Updated", "Should return updated name" + assert body["priority"] == 4, "Should return updated priority" + + async def test_delete_definition_success(self, repo: TaskDefinitionsRepository) -> None: + created = await repo.create(_sample_definition("Delete")) + + request = MagicMock(spec=Request) + request.match_info = {"id": str(created.id)} + + response = await task_definitions_delete(request, Encoder(), MagicMock(spec=EventBus), repo) + assert response.status == web.HTTPOk.status_code, "Should delete task definition" + + async def test_patch_definition_enabled(self, repo: TaskDefinitionsRepository) -> None: + created = await repo.create(_sample_definition("PatchTest", priority=5)) + assert created.enabled is True, "Should be enabled by default" + + request = MagicMock(spec=Request) + request.match_info = {"id": str(created.id)} + request.json = AsyncMock(return_value={"enabled": False}) + + response = await task_definitions_patch(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code, "Should patch task definition" + assert body["name"] == "PatchTest", "Should keep original name" + assert body["priority"] == 5, "Should keep original priority" + assert body["enabled"] is False, "Should update enabled status" + + async def test_patch_definition_priority(self, repo: TaskDefinitionsRepository) -> None: + created = await repo.create(_sample_definition("PatchPriority", priority=1)) + + request = MagicMock(spec=Request) + request.match_info = {"id": str(created.id)} + request.json = AsyncMock(return_value={"priority": 10}) + + response = await task_definitions_patch(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code, "Should patch task definition" + assert body["priority"] == 10, "Should update priority" + assert body["enabled"] is True, "Should keep original enabled status" + + async def test_patch_definition_not_found(self, repo: TaskDefinitionsRepository) -> None: + request = MagicMock(spec=Request) + request.match_info = {"id": "999"} + request.json = AsyncMock(return_value={"enabled": False}) + + response = await task_definitions_patch(request, Encoder(), MagicMock(spec=EventBus), repo) + payload = json.loads(response.text) + + assert response.status == web.HTTPNotFound.status_code, "Should return 404 for missing definition" + assert "error" in payload, "Should include error payload" + + async def test_create_with_regex_pattern(self, repo: TaskDefinitionsRepository) -> None: + payload = _sample_definition("RegexTest", priority=0) + payload["match_url"] = ["/https://example\\.com/post/[0-9]+/"] + + request = MagicMock(spec=Request) + request.json = AsyncMock(return_value=payload) + + response = await task_definitions_create(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPCreated.status_code, "Should create task definition with regex pattern" + assert body["match_url"][0] == "/https://example\\.com/post/[0-9]+/", "Should preserve regex pattern format" + + async def test_create_with_invalid_regex_pattern(self, repo: TaskDefinitionsRepository) -> None: + payload = _sample_definition("BadRegex", priority=0) + payload["match_url"] = ["/[invalid(/"] + + request = MagicMock(spec=Request) + request.json = AsyncMock(return_value=payload) + + response = await task_definitions_create(request, Encoder(), MagicMock(spec=EventBus), repo) + payload_response = json.loads(response.text) + + assert response.status == web.HTTPBadRequest.status_code, "Should reject invalid regex pattern" + assert "error" in payload_response, "Should include error payload" diff --git a/app/tests/test_tver_handler.py b/app/features/tasks/definitions/tests/test_tver_handler.py similarity index 96% rename from app/tests/test_tver_handler.py rename to app/features/tasks/definitions/tests/test_tver_handler.py index b778e6f7..2b8c6442 100644 --- a/app/tests/test_tver_handler.py +++ b/app/features/tasks/definitions/tests/test_tver_handler.py @@ -1,6 +1,6 @@ import pytest -from app.library.task_handlers.tver import TverHandler +from app.features.tasks.definitions.handlers.tver import TverHandler from app.library.Tasks import Task, TaskResult @@ -151,9 +151,10 @@ def test_tver_handler_parse(url: str, should_match: bool): assert result is None -def test_tver_handler_can_handle(): +@pytest.mark.asyncio +async def test_tver_handler_can_handle(): """Test tver handler can_handle method.""" task_valid = Task(id="test1", name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default") task_invalid = Task(id="test2", name="Test", url="https://youtube.com/watch?v=123", preset="default") - assert TverHandler.can_handle(task_valid) is True - assert TverHandler.can_handle(task_invalid) is False + assert await TverHandler.can_handle(task_valid) is True + assert await TverHandler.can_handle(task_invalid) is False diff --git a/app/features/tasks/definitions/utils.py b/app/features/tasks/definitions/utils.py new file mode 100644 index 00000000..1a64dbda --- /dev/null +++ b/app/features/tasks/definitions/utils.py @@ -0,0 +1,48 @@ +from typing import Any + +from app.features.tasks.definitions.models import TaskDefinitionModel +from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary + + +def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary: + """ + Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema. + + Args: + model (TaskDefinitionModel): The model instance to convert. + summary (bool): Whether to return a summary schema. + + Returns: + TaskDefinition | TaskDefinitionSummary: The corresponding schema instance. + + """ + dct = { + "id": model.id, + "name": model.name, + "priority": model.priority, + "match_url": model.match_url, + "enabled": model.enabled, + "created_at": model.created_at, + "updated_at": model.updated_at, + } + return TaskDefinitionSummary(**dct) if summary else TaskDefinition(**dct, definition=Definition(**model.definition)) + + +def schema_to_payload(item: TaskDefinition) -> dict[str, Any]: + """ + Convert a TaskDefinition schema to a dictionary payload for database operations. + + Args: + item (TaskDefinition): The schema instance to convert. + + Returns: + dict[str, Any]: The corresponding dictionary payload. + + """ + return { + "name": item.name, + "priority": item.priority, + "match_url": item.match_url, + "enabled": item.enabled, + "definition": item.definition.model_dump(exclude_unset=True, exclude_none=True), + } diff --git a/app/library/Services.py b/app/library/Services.py index 894c66aa..743fccab 100644 --- a/app/library/Services.py +++ b/app/library/Services.py @@ -1,6 +1,7 @@ import inspect import logging -from typing import Any, TypeVar +from dataclasses import dataclass +from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints from app.library.Singleton import Singleton @@ -8,59 +9,166 @@ T = TypeVar("T") LOG: logging.Logger = logging.getLogger(__name__) +def _unwrap_annotation(ann: Any) -> Any: + if ann is inspect._empty: + return inspect._empty + + origin = get_origin(ann) + + # Annotated[T, ...] -> T + if origin is Annotated: + args = get_args(ann) + return args[0] if args else ann + + # Optional[T] / Union[T, None] / T | None -> T + if origin is None: + return ann + + if str(origin) in ("typing.Union", "types.UnionType"): + args = [a for a in get_args(ann) if a is not type(None)] + if len(args) == 1: + return args[0] + return ann + + return ann + + +@dataclass(frozen=True) +class ServiceEntry: + name: str + declared_type: type | None + instance: Any + + class Services(metaclass=Singleton): def __init__(self): - self._services: dict[str, T] = {} + self._services: list[ServiceEntry] = [] @staticmethod def get_instance() -> "Services": return Services() - def add(self, name: str, service: T): - self._services[name] = service + def add(self, name: str, service: Any, declared_type: type | None = None): + """ + Add a service by name. - def add_all(self, services: dict[str, T]): - for name, service in services.items(): - self.add(name, service) + Args: + name: The name of the service. + service: The service instance. + declared_type: The declared type of the service (optional). - def get(self, name: str) -> T | None: - return self._services.get(name) + """ + if declared_type is None and service is not None: + declared_type = type(service) - def has(self, name: str) -> bool: - return name in self._services + self.remove(name) + self._services.append(ServiceEntry(name=name, declared_type=declared_type, instance=service)) + + def add_all(self, services: dict[str, Any]): + for name, svc in services.items(): + self.add(name, svc) def remove(self, name: str): - if name not in self._services: - return - - self._services.pop(name, None) + self._services = [e for e in self._services if e.name != name] def clear(self): self._services.clear() - def get_all(self) -> dict[str, T]: + def get(self, name: str) -> Any | None: + for e in reversed(self._services): + if e.name == name: + return e.instance + return None + + def has(self, name: str) -> bool: + return any(e.name == name for e in self._services) + + def get_all(self) -> list[ServiceEntry]: return self._services.copy() + def get_by_type(self, expected_type: Any) -> Any | None: + expected_type = _unwrap_annotation(expected_type) + if expected_type is inspect._empty or not isinstance(expected_type, type): + return None + + exact: list[Any] = [e.instance for e in self._services if e.declared_type is expected_type] + if len(exact) == 1: + return exact[0] + if len(exact) > 1: + msg: str = ( + f"Ambiguous dependency for type {expected_type.__name__}: {len(exact)} exact matches. Resolve by name." + ) + raise LookupError(msg) + + candidates: list[Any] = [e.instance for e in self._services if isinstance(e.instance, expected_type)] + if len(candidates) == 0: + return None + + if len(candidates) > 1: + msg: str = ( + f"Ambiguous dependency for type {expected_type.__name__}: " + f"{len(candidates)} candidates. Resolve by name." + ) + raise LookupError(msg) + + return candidates[0] + + def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]: + sig: inspect.Signature = inspect.signature(handler) + + try: + type_hints: dict[str, Any] = get_type_hints(handler) + except Exception: + type_hints = {} + + resolved: dict[str, Any] = {} + + for name, param in sig.parameters.items(): + if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD): + continue + + if name in overrides: + resolved[name] = overrides[name] + continue + + by_name: Any | None = self.get(name) + if by_name is not None: + resolved[name] = by_name + continue + + ann: Any | None = type_hints.get(name, param.annotation) + by_type: Any | None = self.get_by_type(ann) + if by_type is not None: + resolved[name] = by_type + continue + + if param.default is not inspect._empty: + continue + + missing_required: list[str] = [] + for name, param in sig.parameters.items(): + if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD): + continue + + if param.default is not inspect._empty: + continue + + if name not in resolved: + missing_required.append(name) + + if missing_required: + LOG.error( + "Missing arguments for handler '%s': %s", + getattr(handler, "__name__", str(handler)), + missing_required, + ) + + return resolved + async def handle_async(self, handler: callable, **kwargs) -> Any: - context = {**self.get_all(), **kwargs} - - sig = inspect.signature(handler) - expected_args = sig.parameters.keys() - filtered = {k: v for k, v in context.items() if k in expected_args} - - if missing_args := expected_args - filtered.keys(): - LOG.error(f"Missing arguments for handler '{handler.__name__}': {missing_args}") - - return await handler(**filtered) + resolved: dict[str, Any] = self._build_call_args(handler, kwargs) + return await handler(**resolved) def handle_sync(self, handler: callable, **kwargs) -> Any: - context = {**self.get_all(), **kwargs} - - sig = inspect.signature(handler) - expected_args = sig.parameters.keys() - filtered = {k: v for k, v in context.items() if k in expected_args} - - if missing_args := expected_args - filtered.keys(): - LOG.error(f"Missing arguments for handler '{handler.__name__}': {missing_args}") - - return handler(**filtered) + resolved: dict[str, Any] = self._build_call_args(handler, kwargs) + return handler(**resolved) diff --git a/app/library/Tasks.py b/app/library/Tasks.py index b008c733..ad4689e6 100644 --- a/app/library/Tasks.py +++ b/app/library/Tasks.py @@ -689,7 +689,7 @@ class HandleTask: self._scheduler.add( timer=timer, - func=self._dispatcher, + func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"), id=f"{__class__.__name__}._dispatcher", ) @@ -704,7 +704,7 @@ class HandleTask: if self._scheduler.has(self._task_name): self._scheduler.remove(self._task_name) - def _dispatcher(self): + async def _dispatcher(self): s: dict[list[str]] = {"h": [], "d": [], "u": [], "f": []} handler_groups: dict[str, list[tuple[Task, type]]] = {} @@ -720,7 +720,7 @@ class HandleTask: continue try: - handler = self._find_handler(task) + handler = await self._find_handler(task) if handler is None: s["u"].append(task.name) continue @@ -779,10 +779,10 @@ class HandleTask: if exc := fut.exception(): LOG.error(f"Exception while handling task '{task.name}': {exc}") - def _find_handler(self, task: Task) -> type | None: + async def _find_handler(self, task: Task) -> type | None: for cls in self._handlers: try: - if Services.get_instance().handle_sync(handler=cls.can_handle, task=task): + if await Services.get_instance().handle_async(handler=cls.can_handle, task=task): return cls except Exception as e: LOG.exception(e) @@ -804,7 +804,7 @@ class HandleTask: """ if not handler: - handler = self._find_handler(task) + handler = await self._find_handler(task) if handler is None: return None @@ -958,7 +958,7 @@ class HandleTask: ) try: - matched = services.handle_sync(handler=handler_cls.can_handle, task=task) + matched = await services.handle_async(handler=handler_cls.can_handle, task=task) except Exception as exc: # pragma: no cover - defensive LOG.exception(exc) message = str(exc) @@ -974,7 +974,7 @@ class HandleTask: metadata={"matched": False, "handler": handler_cls.__name__}, ) else: - handler_cls = self._find_handler(task) + handler_cls = await self._find_handler(task) if handler_cls is None: message = "No handler matched the supplied URL." return TaskFailure( @@ -1032,7 +1032,7 @@ class HandleTask: def _discover(self) -> list[type]: import importlib - import app.library.task_handlers as handlers_pkg + import app.features.tasks.definitions.handlers as handlers_pkg handlers: list[type] = [] diff --git a/app/main.py b/app/main.py index bbab7563..c050ccd7 100644 --- a/app/main.py +++ b/app/main.py @@ -17,6 +17,7 @@ from aiohttp import web from app.features.conditions.service import Conditions from app.features.dl_fields.service import DLFields from app.features.notifications.service import Notifications +from app.features.tasks.definitions.deps import get_task_definitions_repo from app.library.BackgroundWorker import BackgroundWorker from app.library.cache import Cache from app.library.config import Config @@ -28,7 +29,6 @@ from app.library.Presets import Presets from app.library.Scheduler import Scheduler from app.library.Services import Services from app.library.sqlite_store import SqliteStore -from app.library.TaskDefinitions import TaskDefinitions from app.library.Tasks import Tasks from app.library.UpdateChecker import UpdateChecker @@ -121,7 +121,7 @@ class Main: Notifications.get_instance().attach(self._app) Conditions.get_instance().attach(self._app) DLFields.get_instance().attach(self._app) - TaskDefinitions.get_instance().attach(self._app) + get_task_definitions_repo().attach(self._app) DownloadQueue.get_instance().attach(self._app) UpdateChecker.get_instance().attach(self._app) diff --git a/app/migrations/20260122201858_add_task_definitions.py b/app/migrations/20260122201858_add_task_definitions.py new file mode 100644 index 00000000..927373db --- /dev/null +++ b/app/migrations/20260122201858_add_task_definitions.py @@ -0,0 +1,38 @@ +""" +This module contains a db migration. + +Migration Name: add_task_definitions +Migration Version: 20260122201858 +""" + +from sqlalchemy import text + + +async def upgrade(c): + sql: list[str] = [ + """ + CREATE TABLE IF NOT EXISTS "task_definitions" ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "name" TEXT NOT NULL UNIQUE, + "priority" INTEGER NOT NULL DEFAULT 0, + "enabled" BOOLEAN NOT NULL DEFAULT 1, + "match_url" JSON NOT NULL, + "definition" JSON NOT NULL DEFAULT '{}', + "created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """, + 'CREATE INDEX IF NOT EXISTS "ix_task_definitions_name" ON "task_definitions" ("name");', + 'CREATE INDEX IF NOT EXISTS "ix_task_definitions_priority" ON "task_definitions" ("priority");', + 'CREATE INDEX IF NOT EXISTS "ix_task_definitions_match_url" ON "task_definitions" ("match_url");', + 'CREATE INDEX IF NOT EXISTS "ix_task_definitions_enabled" ON "task_definitions" ("enabled");', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) + + +async def downgrade(c): + sql = """ + DROP TABLE IF EXISTS "task_definitions"; + """ + await c.execute(text(sql)) diff --git a/app/routes/api/task_definitions.py b/app/routes/api/task_definitions.py index bda8f8fb..83dd62c0 100644 --- a/app/routes/api/task_definitions.py +++ b/app/routes/api/task_definitions.py @@ -1,145 +1,3 @@ -import logging -from typing import Any +"""Migrated API routes for feature submodule.""" -from aiohttp import web -from aiohttp.web import Request, Response - -from app.library.encoder import Encoder -from app.library.router import route -from app.library.TaskDefinitions import TaskDefinitionRecord, TaskDefinitions - -LOG: logging.Logger = logging.getLogger(__name__) - - -@route("GET", "api/task_definitions/", "task_definitions") -async def task_definitions_list(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: - include: str | None = request.query.get("include") - include_definition: bool = "definition" == include - - return web.json_response( - data=[item.serialize(include_definition=include_definition) for item in task_definitions.list()], - status=web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("GET", "api/task_definitions/{identifier}", "task_definitions_get") -async def task_definitions_get(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: - identifier: str = request.match_info.get("identifier", "").strip() - if not identifier: - return web.json_response( - data={"error": "Missing task definition identifier."}, - status=web.HTTPBadRequest.status_code, - ) - - record: TaskDefinitionRecord | None = task_definitions.get(identifier) - if not record: - return web.json_response( - data={"error": f"Task definition '{identifier}' not found."}, - status=web.HTTPNotFound.status_code, - ) - - return web.json_response( - data=record.serialize(include_definition=True), - status=web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("POST", "api/task_definitions/", "task_definitions_create") -async def task_definitions_create(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: - try: - payload: Any = await request.json() - if not isinstance(payload, dict): - return web.json_response( - data={"error": "Invalid request body; expected JSON object."}, - status=web.HTTPBadRequest.status_code, - ) - - if "definition" in payload: - if not isinstance(payload["definition"], dict): - return web.json_response( - data={"error": "definition must be a JSON object when provided."}, - status=web.HTTPBadRequest.status_code, - ) - payload = payload["definition"] - - record: TaskDefinitionRecord = task_definitions.create(payload) - except (ValueError, TypeError) as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": "Failed to create task definition."}, - status=web.HTTPInternalServerError.status_code, - ) - - return web.json_response( - data=record.serialize(include_definition=True), - status=web.HTTPCreated.status_code, - dumps=encoder.encode, - ) - - -@route("PUT", "api/task_definitions/{identifier}", "task_definitions_update") -async def task_definitions_update(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: - identifier: str = request.match_info.get("identifier", "").strip() - if not identifier: - return web.json_response( - data={"error": "Missing task definition identifier."}, - status=web.HTTPBadRequest.status_code, - ) - - try: - payload: Any = await request.json() - if not isinstance(payload, dict): - return web.json_response( - data={"error": "Invalid request body; expected JSON object."}, - status=web.HTTPBadRequest.status_code, - ) - - if "definition" in payload: - if not isinstance(payload["definition"], dict): - return web.json_response( - data={"error": "definition must be a JSON object when provided."}, - status=web.HTTPBadRequest.status_code, - ) - payload = payload["definition"] - - record: TaskDefinitionRecord = task_definitions.update(identifier, payload) - except (ValueError, TypeError) as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": "Failed to update task definition."}, - status=web.HTTPInternalServerError.status_code, - ) - - return web.json_response( - data=record.serialize(include_definition=True), - status=web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("DELETE", "api/task_definitions/{identifier}", "task_definitions_delete") -async def task_definitions_delete(request: Request, task_definitions: TaskDefinitions) -> Response: - identifier: str = request.match_info.get("identifier", "").strip() - if not identifier: - return web.json_response( - data={"error": "Missing task definition identifier."}, - status=web.HTTPBadRequest.status_code, - ) - - try: - task_definitions.delete(identifier) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": "Failed to delete task definition."}, status=web.HTTPInternalServerError.status_code - ) - - return web.json_response(data={"status": "deleted"}, status=web.HTTPOk.status_code) +import app.features.tasks.definitions.router # noqa: F401 diff --git a/app/schema/task_definition.json b/app/schema/task_definition.json index 1e3c8917..76a1bd6e 100644 --- a/app/schema/task_definition.json +++ b/app/schema/task_definition.json @@ -5,11 +5,15 @@ "type": "object", "required": [ "name", - "match", - "parse" + "match_url", + "definition" ], "additionalProperties": false, "properties": { + "id": { + "type": "integer", + "description": "Auto-generated unique identifier (read-only)." + }, "name": { "type": "string", "minLength": 1, @@ -21,49 +25,39 @@ "description": "Optional ordering priority. Lower numbers are listed first.", "default": 0 }, - "match": { + "enabled": { + "type": "boolean", + "description": "Whether this definition is active. Disabled definitions are not matched.", + "default": true + }, + "match_url": { "type": "array", "minItems": 1, "items": { - "oneOf": [ - { - "type": "string", - "minLength": 1, - "description": "Glob pattern matched against task URLs." - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "regex": { - "type": "string", - "minLength": 1, - "description": "Regular expression applied to task URLs." - }, - "glob": { - "type": "string", - "minLength": 1, - "description": "Glob pattern applied to task URLs." - } - }, - "anyOf": [ - { - "required": [ - "regex" - ] - }, - { - "required": [ - "glob" - ] - } - ] - } - ] + "type": "string", + "minLength": 1, + "description": "URL pattern. Use glob patterns (e.g. 'https://example.com/*') or regex (e.g. '/https://example\\.com/post/[0-9]+/')." }, - "description": "Patterns that determine which tasks use this definition." + "description": "Patterns that determine which tasks use this definition. Regex patterns are detected by /pattern/ format." }, - "engine": { + "created_at": { + "type": "string", + "format": "date-time", + "description": "Timestamp when definition was created (read-only)." + }, + "updated_at": { + "type": "string", + "format": "date-time", + "description": "Timestamp when definition was last updated (read-only)." + }, + "definition": { + "type": "object", + "required": [ + "parse" + ], + "additionalProperties": false, + "properties": { + "engine": { "type": "object", "additionalProperties": false, "properties": { @@ -123,9 +117,9 @@ } } }, - "description": "Optional engine configuration (defaults to HTTPX)." - }, - "request": { + "description": "Optional engine configuration (defaults to HTTPX)." + }, + "request": { "type": "object", "additionalProperties": false, "properties": { @@ -165,7 +159,7 @@ } ] }, - "json": { + "json_data": { "description": "JSON payload for POST requests.", "type": [ "object", @@ -182,81 +176,93 @@ "description": "Timeout in seconds for the request." } }, - "description": "Optional HTTP request overrides." - }, - "response": { + "description": "Optional HTTP request overrides." + }, + "response": { "type": "object", "additionalProperties": false, "properties": { - "type": { - "type": "string", - "enum": [ - "html", - "json" - ], - "default": "html", - "description": "Body format returned by the target URL." - } - } - }, - "parse": { - "type": "object", - "minProperties": 1, - "description": "Field extraction rules and optional container definition.", - "additionalProperties": false, - "properties": { - "items": { - "$ref": "#/definitions/container" - } - }, - "patternProperties": { - "^(?!items$).+$": { - "$ref": "#/definitions/extractionRule" - } - }, - "allOf": [ - { - "if": { - "not": { - "required": [ - "items" - ] + "type": { + "type": "string", + "enum": [ + "html", + "json" + ], + "default": "html", + "description": "Body format returned by the target URL." + } + } + }, + "parse": { + "type": "object", + "minProperties": 1, + "description": "Field extraction rules and optional container definition.", + "additionalProperties": false, + "properties": { + "items": { + "$ref": "#/definitions/container" } }, - "then": { - "required": [ - "link" - ] - } + "patternProperties": { + "^(?!items$).+$": { + "$ref": "#/definitions/extractionRule" + } + }, + "allOf": [ + { + "if": { + "not": { + "required": [ + "items" + ] + } + }, + "then": { + "required": [ + "link" + ] + } + } + ] } - ] + } } }, "allOf": [ { "if": { "properties": { - "engine": { + "definition": { "properties": { - "type": { "const": "selenium" } + "engine": { + "properties": { + "type": { "const": "selenium" } + }, + "required": ["type"] + } }, - "required": ["type"] + "required": ["engine"] } }, - "required": ["engine"] + "required": ["definition"] }, "then": { "properties": { - "engine": { + "definition": { "properties": { - "options": { - "required": ["url"] + "engine": { + "properties": { + "options": { + "required": ["url"] + } + }, + "required": ["options"] } }, - "required": ["options"] + "required": ["engine"] } }, - "required": ["engine"] + "required": ["definition"] } } ], @@ -421,66 +427,68 @@ "examples": [ { "name": "example", - "match": [ + "match_url": [ "https://example.com/articles/*", - { - "regex": "https://example.com/post/[0-9]+" - } + "/https://example\\.com/post/[0-9]+/" ], - "engine": { - "type": "httpx" - }, - "request": { - "method": "GET", - "headers": { - "User-Agent": "MyCustomAgent/1.0" - } - }, - "parse": { - "link": { - "type": "css", - "expression": ".article a.link", - "attribute": "href" + "definition": { + "engine": { + "type": "httpx" }, - "title": { - "type": "css", - "expression": ".article .title", - "attribute": "text" + "request": { + "method": "GET", + "headers": { + "User-Agent": "MyCustomAgent/1.0" + } }, - "id": { - "type": "regex", - "expression": "id=(?P[0-9]+)", - "post_filter": { - "filter": "(?P[0-9]+)", - "value": "id" + "parse": { + "link": { + "type": "css", + "expression": ".article a.link", + "attribute": "href" + }, + "title": { + "type": "css", + "expression": ".article .title", + "attribute": "text" + }, + "id": { + "type": "regex", + "expression": "id=(?P[0-9]+)", + "post_filter": { + "filter": "(?P[0-9]+)", + "value": "id" + } } } } }, { "name": "container-example", - "match": [ + "match_url": [ "https://example.com/list" ], - "parse": { - "items": { - "type": "css", - "selector": ".cards .card", - "fields": { - "link": { - "type": "css", - "expression": ".card-header a", - "attribute": "href" - }, - "title": { - "type": "css", - "expression": ".card-header a", - "attribute": "text" - }, - "poet": { - "type": "css", - "expression": "footer .card-footer-item:first-child a", - "attribute": "text" + "definition": { + "parse": { + "items": { + "type": "css", + "selector": ".cards .card", + "fields": { + "link": { + "type": "css", + "expression": ".card-header a", + "attribute": "href" + }, + "title": { + "type": "css", + "expression": ".card-header a", + "attribute": "text" + }, + "poet": { + "type": "css", + "expression": "footer .card-footer-item:first-child a", + "attribute": "text" + } } } } diff --git a/app/tests/test_generic_task_handler.py b/app/tests/test_generic_task_handler.py deleted file mode 100644 index 6777bd0c..00000000 --- a/app/tests/test_generic_task_handler.py +++ /dev/null @@ -1,362 +0,0 @@ -import json -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -import pytest - -from app.library.task_handlers.generic import ( - ContainerDefinition, - EngineConfig, - ExtractionRule, - GenericTaskHandler, - RequestConfig, - ResponseConfig, - TaskDefinition, - load_task_definitions, -) -from app.library.Tasks import Task, TaskFailure, TaskResult - - -@pytest.fixture(autouse=True) -def reset_generic_handler(monkeypatch): - monkeypatch.setattr(GenericTaskHandler, "_definitions", []) - monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {}) - - -def test_load_task_definitions_parses_valid_file(tmp_path: Path): - tasks_dir = tmp_path / "tasks" - tasks_dir.mkdir() - - definition_content = { - "name": "example", - "match": ["https://example.com/articles/*"], - "parse": { - "link": {"type": "css", "expression": ".article a.link::attr(href)"}, - "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, - }, - } - - (tasks_dir / "01-example.json").write_text(json.dumps(definition_content), encoding="utf-8") - - config = SimpleNamespace(config_path=str(tmp_path)) - definitions = load_task_definitions(config=config) - - assert len(definitions) == 1 - definition = definitions[0] - assert definition.name == "example" - assert definition.matchers[0].matches("https://example.com/articles/123") - assert definition.parsers["link"].expression == ".article a.link::attr(href)" - - -def test_load_task_definitions_handles_container(tmp_path: Path): - tasks_dir = tmp_path / "tasks" - tasks_dir.mkdir() - - definition_content = { - "name": "container", - "match": ["https://example.com/cards"], - "parse": { - "items": { - "selector": ".cards .card", - "fields": { - "link": {"type": "css", "expression": ".card-header a", "attribute": "href"}, - "title": {"type": "css", "expression": ".card-header a", "attribute": "text"}, - }, - } - }, - } - - (tasks_dir / "02-container.json").write_text(json.dumps(definition_content), encoding="utf-8") - - config = SimpleNamespace(config_path=str(tmp_path)) - definitions = load_task_definitions(config=config) - - assert len(definitions) == 1 - definition = definitions[0] - assert definition.container is not None - assert definition.container.selector == ".cards .card" - assert "link" in definition.container.fields - assert definition.parsers == {} - - -def test_load_task_definitions_handles_json(tmp_path: Path): - tasks_dir = tmp_path / "tasks" - tasks_dir.mkdir() - - definition_content = { - "name": "json-def", - "match": ["https://example.com/api"], - "response": {"type": "json"}, - "parse": { - "items": { - "type": "jsonpath", - "selector": "items", - "fields": { - "link": {"type": "jsonpath", "expression": "url"}, - "title": {"type": "jsonpath", "expression": "title"}, - }, - } - }, - } - - (tasks_dir / "03-json.json").write_text(json.dumps(definition_content), encoding="utf-8") - - config = SimpleNamespace(config_path=str(tmp_path)) - definitions = load_task_definitions(config=config) - - assert len(definitions) == 1 - definition = definitions[0] - assert definition.response.format == "json" - assert definition.container is not None - assert definition.container.selector_type == "jsonpath" - assert definition.container.fields["link"].type == "jsonpath" - - -def test_parse_items_extracts_values(): - definition = TaskDefinition( - name="example", - source=Path("example.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={ - "link": ExtractionRule(type="css", expression=".article a.link::attr(href)", attribute=None), - "title": ExtractionRule(type="css", expression=".article .title", attribute="text"), - "id": ExtractionRule(type="css", expression=".article", attribute="data-id"), - }, - ) - - html = """ -
- First - First Title -
-
- Second - Second Title -
- """ - - items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/") - - assert len(items) == 2 - assert items[0]["link"] == "https://example.com/article-101" - assert items[0]["title"] == "First Title" - assert items[0]["id"] == "101" - assert items[1]["link"] == "https://example.com/article-102" - - -def test_parse_items_handles_nested_card_layout(): - definition = TaskDefinition( - name="nested", - source=Path("nested.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={}, - container=ContainerDefinition( - selector_type="css", - selector=".columns .card", - fields={ - "link": ExtractionRule( - type="css", - expression=".card-header a[href]", - attribute="href", - ), - "title": ExtractionRule( - type="css", - expression=".card-header a[href]", - attribute="text", - ), - "poet": ExtractionRule( - type="css", - expression="footer .card-footer-item:first-child a", - attribute="text", - ), - "category": ExtractionRule( - type="css", - expression="footer .card-footer-item:nth-child(2) a", - attribute="text", - ), - }, - ), - ) - - html = """ -
-
-
-
-

- First Poem -

-
- -
-
-
-
-
-

- Second Poem -

-
- -
-
-
- """ - - items = GenericTaskHandler._parse_items(definition, html, "https://example.com") - - assert len(items) == 2 - assert items[0]["link"] == "https://example.com/poems/view/111" - assert items[0]["title"] == "First Poem" - assert items[0]["poet"] == "Poet Alpha" - assert items[0]["category"] == "Category One" - - assert items[1]["link"] == "https://example.com/poems/view/222" - assert items[1]["title"] == "Second Poem" - assert items[1]["poet"] == "Poet Beta" - assert "category" not in items[1] - - -def test_parse_items_handles_json_container(): - definition = TaskDefinition( - name="json", - source=Path("json.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={}, - container=ContainerDefinition( - selector_type="jsonpath", - selector="entries", - fields={ - "link": ExtractionRule(type="jsonpath", expression="url"), - "title": ExtractionRule(type="jsonpath", expression="title"), - "id": ExtractionRule(type="jsonpath", expression="id"), - }, - ), - response=ResponseConfig(format="json"), - ) - - payload = { - "entries": [ - {"url": "/video/1", "title": "First", "id": 1}, - {"url": "https://example.com/video/2", "title": "Second", "id": 2}, - {"title": "Missing Link", "id": 3}, - ] - } - - items = GenericTaskHandler._parse_items( - definition=definition, - html="", - base_url="https://example.com", - json_data=payload, - ) - - assert len(items) == 2 - assert items[0]["link"] == "https://example.com/video/1" - assert items[0]["title"] == "First" - assert items[0]["id"] == "1" - - assert items[1]["link"] == "https://example.com/video/2" - assert items[1]["title"] == "Second" - assert items[1]["id"] == "2" - - -@pytest.mark.asyncio -async def test_generic_task_handler_inspect(monkeypatch): - definition = TaskDefinition( - name="json-inspect", - source=Path("json-inspect.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={}, - container=ContainerDefinition( - selector_type="jsonpath", - selector="items", - fields={ - "link": ExtractionRule(type="jsonpath", expression="url"), - "title": ExtractionRule(type="jsonpath", expression="title"), - }, - ), - response=ResponseConfig(format="json"), - ) - - monkeypatch.setattr( - GenericTaskHandler, - "_find_definition", - classmethod(lambda cls, url: definition), # noqa: ARG005 - ) - - async def fake_fetch_content(url, definition, ytdlp_opts): # noqa: ARG001 - return "", {"items": [{"url": "/video/1", "title": "First"}]} - - monkeypatch.setattr(GenericTaskHandler, "_fetch_content", staticmethod(fake_fetch_content)) - - # Mock fetch_info to return valid info with required fields for archive ID generation - async def fake_fetch_info(config, url, **kwargs): # noqa: ARG001 - return {"id": "test_video_1", "extractor_key": "Example"} - - with patch("app.library.task_handlers.generic.fetch_info", side_effect=fake_fetch_info): - task = Task(id="inspect", name="Inspect", url="https://example.com/api") - result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task) - - assert isinstance(result, TaskResult) - assert len(result.items) == 1 - item = result.items[0] - assert item.url == "https://example.com/video/1" - assert item.title == "First" - - -def test_parse_items_handles_json_top_level_list(): - definition = TaskDefinition( - name="json-list", - source=Path("json-list.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={}, - container=ContainerDefinition( - selector_type="jsonpath", - selector="[]", - fields={ - "link": ExtractionRule(type="jsonpath", expression="url"), - "title": ExtractionRule(type="jsonpath", expression="title"), - }, - ), - response=ResponseConfig(format="json"), - ) - - payload = [ - {"url": "/video/1", "title": "First"}, - {"url": "/video/2", "title": "Second"}, - ] - - items = GenericTaskHandler._parse_items( - definition=definition, - html="", - base_url="https://example.com", - json_data=payload, - ) - - assert len(items) == 2 - assert items[0]["link"] == "https://example.com/video/1" - assert items[0]["title"] == "First" - assert items[1]["link"] == "https://example.com/video/2" - assert items[1]["title"] == "Second" diff --git a/app/tests/test_services.py b/app/tests/test_services.py index be64623a..e151bba6 100644 --- a/app/tests/test_services.py +++ b/app/tests/test_services.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock, patch import pytest -from app.library.Services import Services +from app.library.Services import ServiceEntry, Services class TestServices: @@ -92,9 +92,9 @@ class TestServices: services.add("test", "value") all_services = services.get_all() - all_services["injected"] = "malicious" + all_services.append(ServiceEntry(name="injected", declared_type=str, instance="malicious")) - assert "injected" not in services.get_all(), "Modifying returned dict should not affect internal state" + assert services.get("injected") is None, "Modifying returned dict should not affect internal state" def test_handle_sync_with_matching_args(self): """Test synchronous handler with matching arguments.""" @@ -138,8 +138,9 @@ class TestServices: # Should still log error about missing arguments mock_logger.error.assert_called_once() error_call = mock_logger.error.call_args[0][0] - assert "Missing arguments" in error_call - assert "missing_service_param" in error_call + assert "Missing arguments for handler" in error_call, ( + f"Expected 'Missing arguments for handler' in log, got: {error_call}" + ) def test_handle_sync_no_args_handler(self): """Test synchronous handler that takes no arguments.""" @@ -197,8 +198,9 @@ class TestServices: # Should still log error about missing arguments mock_logger.error.assert_called_once() error_call = mock_logger.error.call_args[0][0] - assert "Missing arguments" in error_call - assert "missing_service_param" in error_call + assert "Missing arguments for handler" in error_call, ( + f"Expected 'Missing arguments for handler' in log, got: {error_call}" + ) @pytest.mark.asyncio async def test_handle_async_no_args_handler(self): diff --git a/app/tests/test_task_definitions.py b/app/tests/test_task_definitions.py deleted file mode 100644 index 28a42bb8..00000000 --- a/app/tests/test_task_definitions.py +++ /dev/null @@ -1,315 +0,0 @@ -import json -import uuid -from pathlib import Path -from typing import Any -from unittest.mock import AsyncMock, MagicMock, Mock, patch - -import pytest -from aiohttp import web -from aiohttp.web import Request -from jsonschema import Draft7Validator - -from app.library.encoder import Encoder -from app.library.TaskDefinitions import TaskDefinitionRecord, TaskDefinitions -from app.routes.api import task_definitions as api - - -def _load_validator() -> Draft7Validator: - schema_path = Path(__file__).resolve().parent.parent / "schema" / "task_definition.json" - schema = json.loads(schema_path.read_text(encoding="utf-8")) - return Draft7Validator(schema) - - -def _sample_definition(name: str = "example", *, priority: int = 0) -> dict[str, Any]: - return { - "name": name, - "match": ["https://example.com/*"], - "priority": priority, - "parse": { - "items": { - "type": "css", - "selector": ".card", - "fields": { - "link": {"type": "css", "expression": "a", "attribute": "href"}, - "title": {"type": "css", "expression": "a", "attribute": "text"}, - }, - } - }, - } - - -class TestTaskDefinitionsManager: - def setup_method(self) -> None: - TaskDefinitions._reset_singleton() - - def teardown_method(self) -> None: - TaskDefinitions._reset_singleton() - - def test_load_populates_records(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - - first_identifier = "b5c6ad5f-4745-4c05-88c8-dde1deae3b51" - second_identifier = "ae38a6b0-2c22-4763-ba60-801ae8ce1218" - - first = tmp_path / f"{first_identifier}.json" - second = tmp_path / f"{second_identifier}.json" - - first.write_text(json.dumps(_sample_definition("First", priority=5)), encoding="utf-8") - second.write_text(json.dumps(_sample_definition("Second", priority=1)), encoding="utf-8") - - (tmp_path / "not-a-uuid.json").write_text(json.dumps(_sample_definition("Ignored")), encoding="utf-8") - (tmp_path / "0f9184de-5d3c-2111-8c21-6d3f0be1bd3d.json").write_text( - json.dumps(_sample_definition("Ignored2")), - encoding="utf-8", - ) - - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - records = manager.list() - assert len(records) == 2 - assert [record.name for record in records] == ["Second", "First"] - assert [record.priority for record in records] == [1, 5] - - def test_create_writes_file_and_refreshes(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh: - definition = _sample_definition("My Definition") - record = manager.create(definition) - - refresh.assert_called_once_with(force=True) - identifier_uuid = uuid.UUID(record.identifier) - assert 4 == identifier_uuid.version - expected_filename = f"{record.identifier}.json" - saved_path = tmp_path / expected_filename - assert saved_path.exists() - saved_content = json.loads(saved_path.read_text(encoding="utf-8")) - assert saved_content["name"] == "My Definition" - assert saved_content["priority"] == 0 - - def test_update_missing_definition_raises(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - with pytest.raises(ValueError, match="does not exist"): - manager.update("missing", _sample_definition("Updated")) - - def test_update_overwrites_file(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - - identifier = "c59ec7cf-6291-4f0f-86f8-d8cb12c325a4" - initial = _sample_definition("Original", priority=4) - path = tmp_path / f"{identifier}.json" - path.write_text(json.dumps(initial), encoding="utf-8") - - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh: - updated_record = manager.update(identifier, _sample_definition("Updated", priority=2)) - - refresh.assert_called_once_with(force=True) - assert updated_record.name == "Updated" - assert updated_record.priority == 2 - saved = json.loads(path.read_text(encoding="utf-8")) - assert saved["name"] == "Updated" - assert saved["priority"] == 2 - - def test_delete_removes_file(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - - identifier = "f0b71f47-6b65-4b6d-89fd-6b87ce47d3bc" - definition_path = tmp_path / f"{identifier}.json" - definition_path.write_text(json.dumps(_sample_definition("Delete")), encoding="utf-8") - - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh: - manager.delete(identifier) - - refresh.assert_called_once_with(force=True) - assert not definition_path.exists() - assert manager.get(identifier) is None - - -@pytest.mark.asyncio -class TestTaskDefinitionRoutes: - def setup_method(self) -> None: - TaskDefinitions._reset_singleton() - - def teardown_method(self) -> None: - TaskDefinitions._reset_singleton() - - async def test_list_definitions(self) -> None: - request = MagicMock(spec=Request) - request.query = {} - - identifier = "9af7018f-8659-4d2a-a42b-b5d2c5f0a6e2" - record = TaskDefinitionRecord( - identifier=identifier, - filename=f"{identifier}.json", - name="Sample", - priority=0, - path=Path("/tmp/sample.json"), - data=_sample_definition("Sample"), - updated_at=123.0, - ) - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.list.return_value = [record] - - response = await api.task_definitions_list(request, Encoder(), task_definitions) - payload = json.loads(response.text) - - assert response.status == web.HTTPOk.status_code - assert payload == [record.serialize()] - assert "filename" not in payload[0] - - async def test_list_definitions_includes_definition(self) -> None: - request = MagicMock(spec=Request) - request.query = {"include": "definition"} - - identifier = "f5b5e88d-5c6b-4a27-8453-1b6a4fb8a8d1" - record = TaskDefinitionRecord( - identifier=identifier, - filename=f"{identifier}.json", - name="Sample", - priority=0, - path=Path("/tmp/sample.json"), - data=_sample_definition("Sample"), - updated_at=123.0, - ) - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.list.return_value = [record] - - response = await api.task_definitions_list(request, Encoder(), task_definitions) - payload = json.loads(response.text) - - assert payload[0]["definition"]["name"] == "Sample" - - async def test_get_definition_not_found(self) -> None: - request = MagicMock(spec=Request) - request.match_info = {"identifier": "unknown"} - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.get.return_value = None - - response = await api.task_definitions_get(request, Encoder(), task_definitions) - payload = json.loads(response.text) - - assert response.status == web.HTTPNotFound.status_code - assert "error" in payload - - async def test_create_definition_success(self) -> None: - payload_definition = _sample_definition("New", priority=1) - payload = {"definition": payload_definition} - - request = MagicMock(spec=Request) - request.json = AsyncMock(return_value=payload) - - identifier = "4f08b8af-b87a-4d6e-9289-39e5172898aa" - record = TaskDefinitionRecord( - identifier=identifier, - filename=f"{identifier}.json", - name="New", - priority=1, - path=Path("/tmp/new.json"), - data=payload["definition"], - updated_at=123.0, - ) - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.create.return_value = record - - response = await api.task_definitions_create(request, Encoder(), task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPCreated.status_code - assert body["id"] == identifier - assert body["priority"] == 1 - assert "filename" not in body - task_definitions.create.assert_called_once_with(payload_definition) - - async def test_create_definition_invalid_payload(self) -> None: - request = MagicMock(spec=Request) - request.json = AsyncMock(return_value=[]) # type: ignore[arg-type] - - task_definitions = MagicMock(spec=TaskDefinitions) - - response = await api.task_definitions_create(request, Encoder(), task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPBadRequest.status_code - assert "error" in body - - async def test_update_definition_success(self) -> None: - request = MagicMock(spec=Request) - identifier = "6d8d5719-95ae-4478-bb05-986f5b72b6c1" - request.match_info = {"identifier": identifier} - definition = _sample_definition("Updated", priority=4) - request.json = AsyncMock(return_value={"definition": definition}) - record = TaskDefinitionRecord( - identifier=identifier, - filename=f"{identifier}.json", - name="Updated", - priority=4, - path=Path("/tmp/existing.json"), - data=definition, - updated_at=456.0, - ) - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.update.return_value = record - - response = await api.task_definitions_update(request, Encoder(), task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPOk.status_code - assert body["name"] == "Updated" - assert body["priority"] == 4 - task_definitions.update.assert_called_once_with(identifier, definition) - - async def test_update_definition_missing_identifier(self) -> None: - request = MagicMock(spec=Request) - request.match_info = {"identifier": ""} - request.json = AsyncMock(return_value={}) - - task_definitions = MagicMock(spec=TaskDefinitions) - - response = await api.task_definitions_update(request, Encoder(), task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPBadRequest.status_code - assert "error" in body - - async def test_delete_definition_success(self) -> None: - request = MagicMock(spec=Request) - identifier = "c9f4ac6c-a4ab-4d1a-8d25-764dc0c8a3f0" - request.match_info = {"identifier": identifier} - - task_definitions = MagicMock(spec=TaskDefinitions) - - response = await api.task_definitions_delete(request, task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPOk.status_code - assert body["status"] == "deleted" - task_definitions.delete.assert_called_once_with(identifier) - - async def test_delete_definition_missing_identifier(self) -> None: - request = MagicMock(spec=Request) - request.match_info = {"identifier": ""} - - task_definitions = MagicMock(spec=TaskDefinitions) - - response = await api.task_definitions_delete(request, task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPBadRequest.status_code - assert "error" in body diff --git a/app/tests/test_tasks.py b/app/tests/test_tasks.py index 41e5d65c..5d3dcb55 100644 --- a/app/tests/test_tasks.py +++ b/app/tests/test_tasks.py @@ -1053,7 +1053,8 @@ class TestHandleTaskInspect: # Mock Services to simulate successful can_handle mock_services_instance = Mock() mock_services_instance.handle_sync = Mock(return_value=True) - mock_services_instance.handle_async = AsyncMock() # Should NOT be called with static_only=True + # handle_async will be called once for can_handle (since it's now async) + mock_services_instance.handle_async = AsyncMock(return_value=True) mock_services.get_instance.return_value = mock_services_instance # Create a mock handler @@ -1077,8 +1078,8 @@ class TestHandleTaskInspect: assert result.metadata["matched"] is True assert result.metadata["handler"] == "TestHandler" - # Verify handle_async (extract) was NOT called - mock_services_instance.handle_async.assert_not_called() + # Verify handle_async was called once for can_handle (now async) + assert mock_services_instance.handle_async.call_count == 1, "Should call handle_async once for can_handle" @pytest.mark.asyncio @patch("app.library.Tasks.Config") @@ -1132,8 +1133,10 @@ class TestHandleTaskInspect: assert result.metadata["handler"] == "TestHandler" assert result.metadata["supported"] is True - # Verify handle_async (extract) WAS called - mock_services_instance.handle_async.assert_called_once() + # Verify handle_async was called twice: once for can_handle, once for extract + assert mock_services_instance.handle_async.call_count == 2, ( + "Should call handle_async twice: can_handle + extract" + ) @pytest.mark.asyncio @patch("app.library.Tasks.Config") diff --git a/app/tests/test_twitch_handler.py b/app/tests/test_twitch_handler.py index 821eb605..e5b0fc86 100644 --- a/app/tests/test_twitch_handler.py +++ b/app/tests/test_twitch_handler.py @@ -1,6 +1,6 @@ import pytest -from app.library.task_handlers.twitch import TwitchHandler +from app.features.tasks.definitions.handlers.twitch import TwitchHandler from app.library.Tasks import Task, TaskResult diff --git a/app/tests/test_youtube_handler.py b/app/tests/test_youtube_handler.py index 26ee9c91..26c49db1 100644 --- a/app/tests/test_youtube_handler.py +++ b/app/tests/test_youtube_handler.py @@ -1,6 +1,6 @@ import pytest -from app.library.task_handlers.youtube import YoutubeHandler +from app.features.tasks.definitions.handlers.youtube import YoutubeHandler from app.library.Tasks import Task, TaskResult diff --git a/pyproject.toml b/pyproject.toml index a618c6d3..398d8c23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -186,7 +186,7 @@ pythonpath = ["."] testpaths = ["app/tests", "app/features"] addopts = "-v --tb=short" filterwarnings = [ - "ignore:Parsing dates involving a day of month without a year:DeprecationWarning", + "ignore:Parsing dates involving a day of month without a year:DeprecationWarning" ] [dependency-groups] diff --git a/ui/app/components/TaskDefinitionEditor.vue b/ui/app/components/TaskDefinitionEditor.vue index 28192fb1..d97499e9 100644 --- a/ui/app/components/TaskDefinitionEditor.vue +++ b/ui/app/components/TaskDefinitionEditor.vue @@ -32,7 +32,7 @@
-
+
-
+
@@ -125,6 +126,25 @@
+
+
+ +
+ +
+

+ + Disabled definitions won't match tasks. +

+
+
+

- One glob per line. Regex or object based rules are only supported in advanced mode. + One glob/regex url per line

@@ -377,6 +397,7 @@ type GuiField = { type GuiState = { name: string priority: number + enabled: boolean matchText: string engineType: 'httpx' | 'selenium' engineUrl: string @@ -399,7 +420,7 @@ const props = defineProps<{ const emit = defineEmits<{ (e: 'submit', payload: TaskDefinitionDocument): void (e: 'cancel'): void - (e: 'import-existing', id: string): void + (e: 'import-existing', id: number): void }>() const jsonText = ref('') @@ -409,13 +430,14 @@ const guiSupported = ref(true) const mode = ref('gui') const showImport = ref(false) const importString = ref('') -const selectedExisting = ref('') +const selectedExisting = ref('') const availableDefinitions = computed(() => props.availableDefinitions ?? []) const guiState = reactive({ name: '', priority: 0, + enabled: true, matchText: '', engineType: 'httpx', engineUrl: '', @@ -431,12 +453,13 @@ const submitting = computed(() => props.submitting ?? false) const isBusy = computed(() => loading.value || submitting.value) const headerTitle = computed(() => props.title) -const guiLimitations = 'Only simple match globs, a single container selector and per-field extractors are exposed. ' + +const guiLimitations = 'Only a single container selector and per-field extractors are exposed. ' + 'More advanced constructs require raw view mode.' const resetGuiState = (state: GuiState): void => { guiState.name = state.name guiState.priority = state.priority + guiState.enabled = state.enabled guiState.matchText = state.matchText guiState.engineType = state.engineType guiState.engineUrl = state.engineUrl @@ -467,12 +490,17 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => { } const entry = document - const match = entry.match + const match = entry.match_url if (!Array.isArray(match) || match.some(item => 'string' !== typeof item)) { return null } - const parse = entry.parse + const definition = entry.definition + if (!definition || Array.isArray(definition) || 'object' !== typeof definition) { + return null + } + + const parse = definition.parse if (!parse || Array.isArray(parse) || 'object' !== typeof parse) { return null } @@ -499,7 +527,7 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => { if ('string' !== typeof rule.type || 'string' !== typeof rule.expression) { return null } - if (Object.keys(rule).some(prop => !['type', 'expression', 'attribute'].includes(prop))) { + if (Object.keys(rule).some(prop => !['type', 'expression', 'attribute', 'post_filter'].includes(prop))) { return null } guiFields.push({ @@ -510,7 +538,7 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => { }) } - const engine = entry.engine as Record | undefined + const engine = definition.engine as Record | undefined const engineType = (engine?.type === 'selenium') ? 'selenium' : 'httpx' const engineUrl = 'string' === typeof engine?.options && engineType === 'selenium' ? '' @@ -520,7 +548,7 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => { return null } - const request = entry.request as Record | undefined + const request = definition.request as Record | undefined const selectorType = String(itemRecord.type ?? 'css') as GuiState['containerType'] const selectorSource = (itemRecord.selector ?? itemRecord.expression) as string | undefined @@ -531,6 +559,7 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => { return { name: 'string' === typeof entry.name ? entry.name : '', priority: Number(entry.priority ?? 0) || 0, + enabled: 'boolean' === typeof entry.enabled ? entry.enabled : true, matchText: match.join('\n'), engineType, engineUrl: engineType === 'selenium' ? String(engineUrl ?? '') : '', @@ -575,10 +604,7 @@ const fromGui = (state: GuiState): TaskDefinitionDocument => { throw new Error('Configure at least one extractor field.') } - const doc: Record = { - name: state.name.trim(), - priority: Number(state.priority) || 0, - match: matches, + const definition: Record = { parse: { items: { type: state.containerType, @@ -590,7 +616,7 @@ const fromGui = (state: GuiState): TaskDefinitionDocument => { } if ('httpx' !== state.engineType || state.engineUrl) { - doc.engine = { + definition.engine = { type: state.engineType, ...(state.engineType === 'selenium' && state.engineUrl ? { options: { url: state.engineUrl } } @@ -606,10 +632,31 @@ const fromGui = (state: GuiState): TaskDefinitionDocument => { request.url = state.requestUrl } if (Object.keys(request).length) { - doc.request = request + definition.request = request } - return doc as unknown as TaskDefinitionDocument + return { + name: state.name.trim(), + priority: Number(state.priority) || 0, + enabled: state.enabled, + match_url: matches, + definition: definition as unknown as TaskDefinitionDocument['definition'], + } +} + +const normalizeRequestConfig = (request: any): any => { + if (!request || 'object' !== typeof request) { + return request + } + + if ('json' in request) { + const normalized = { ...request } + normalized.json_data = normalized.json + delete normalized.json + return normalized + } + + return request } const parseImportedDocument = (payload: unknown): TaskDefinitionDocument => { @@ -618,31 +665,59 @@ const parseImportedDocument = (payload: unknown): TaskDefinitionDocument => { } const record = payload as Record - const candidate = record.definition + if ('_type' in record && record._type !== undefined && record._type !== 'task_definition') { throw new Error('Import string is not a task definition export.') } + + const version = record._version as string | undefined + if (-1 === ['1.0', '2.0'].indexOf(version ?? '')) { + throw new Error(`Unsupported or missing _version field. Expected "1.0" or "2.0", got: ${version ?? 'undefined'}`) + } + let base: TaskDefinitionDocument - if (candidate && !Array.isArray(candidate) && 'object' === typeof candidate) { - base = candidate as TaskDefinitionDocument + if ('1.0' === version) { + // v1.0 format migration + const oldDef = record.definition as Record + + // Normalize match_url from old v1.0 format + const oldMatch = Array.isArray(oldDef.match) ? oldDef.match : [] + const normalizedMatch: string[] = [] + + for (const item of oldMatch) { + if ('string' === typeof item) { + normalizedMatch.push(item) + } + else if ('object' === typeof item && item !== null) { + const obj = item as Record + if ('string' === typeof obj.regex) { + normalizedMatch.push(`/${obj.regex}/`) + } + else if ('string' === typeof obj.glob) { + normalizedMatch.push(obj.glob) + } + } + } + + base = { + name: 'string' === typeof oldDef.name ? oldDef.name : 'string' === typeof record.name ? record.name : '', + priority: Number(oldDef.priority ?? record.priority ?? 0) || 0, + enabled: true, + match_url: normalizedMatch, + definition: { + parse: oldDef.parse as any, + engine: oldDef.engine as any, + request: normalizeRequestConfig(oldDef.request), + response: oldDef.response as any, + }, + } } else { - base = payload as TaskDefinitionDocument + base = record as unknown as TaskDefinitionDocument } - const clone = JSON.parse(JSON.stringify(base)) as TaskDefinitionDocument - const cloneRecord = clone - - if ('name' in record && 'string' === typeof record.name) { - cloneRecord.name = record.name - } - - if ('priority' in record && record.priority !== undefined) { - cloneRecord.priority = Number(record.priority) || 0 - } - - return clone + return JSON.parse(JSON.stringify(base)) as TaskDefinitionDocument } const parseDocument = (): TaskDefinitionDocument | null => { @@ -675,6 +750,7 @@ const applyDocument = (document: TaskDefinitionDocument | null): void => { resetGuiState({ name: '', priority: 0, + enabled: true, matchText: '', engineType: 'httpx', engineUrl: '', @@ -741,7 +817,7 @@ const importExisting = (): void => { return } - emit('import-existing', selectedExisting.value) + emit('import-existing', Number(selectedExisting.value)) selectedExisting.value = '' } diff --git a/ui/app/composables/useTaskDefinitions.ts b/ui/app/composables/useTaskDefinitions.ts index 39afd5ad..8a50e282 100644 --- a/ui/app/composables/useTaskDefinitions.ts +++ b/ui/app/composables/useTaskDefinitions.ts @@ -1,18 +1,29 @@ import { ref, readonly } from 'vue' import { useNotification } from '~/composables/useNotification' -import { request } from '~/utils' +import { request, parse_list_response, parse_api_response, parse_api_error } from '~/utils' import type { TaskDefinitionDetailed, TaskDefinitionDocument, - TaskDefinitionErrorResponse, TaskDefinitionSummary, } from '~/types/task_definitions' +import type { Pagination } from '~/types/responses' /** * Reactive list of all task definition summaries, sorted by priority and name. */ const definitions = ref>([]) +/** + * Pagination state for task definitions list. + */ +const pagination = ref({ + page: 1, + per_page: 50, + total: 0, + total_pages: 0, + has_next: false, + has_prev: false, +}) /** * Indicates if a request is in progress. */ @@ -47,21 +58,6 @@ const sortSummaries = (items: Array): Array => { - try { - const clone = response.clone() - return await clone.json() - } - catch { - return null - } -} - /** * Throws an error if the response is not OK, using API error message if available. * @param response Fetch Response object @@ -72,16 +68,8 @@ const ensureSuccess = async (response: Response): Promise => { return } - const payload = await readJson(response) - - let message = `Request failed with status ${response.status}` - if (payload && typeof payload === 'object' && 'error' in payload) { - const errorPayload = payload as TaskDefinitionErrorResponse - if (typeof errorPayload.error === 'string' && errorPayload.error.length > 0) { - message = errorPayload.error - } - } - + const payload = await response.clone().json().catch(() => null) + const message = await parse_api_error(payload) throw new Error(message) } @@ -100,48 +88,47 @@ const handleError = (error: unknown): void => { * @param summary TaskDefinitionSummary to update/add */ const updateSummaries = (summary: TaskDefinitionSummary): void => { + const isNew = !definitions.value.some(item => item.id === summary.id) definitions.value = sortSummaries([ ...definitions.value.filter(item => item.id !== summary.id), summary, ]) + if (isNew) { + pagination.value.total++ + } } /** * Removes a summary from the definitions list by ID. * @param id Task definition ID */ -const removeSummary = (id: string) => definitions.value = definitions.value.filter(item => item.id !== id) +const removeSummary = (id: number) => { + const initialLength = definitions.value.length + definitions.value = definitions.value.filter(item => item.id !== id) + if (definitions.value.length < initialLength) { + pagination.value.total = Math.max(0, pagination.value.total - 1) + } +} /** * Loads all task definition summaries from the API. * Updates definitions and lastError. */ -const loadDefinitions = async (): Promise => { +const loadDefinitions = async (page: number = 1, perPage: number | undefined = undefined): Promise => { isLoading.value = true try { - const response = await request('/api/task_definitions/') + let url = `/api/tasks/definitions/?page=${page}` + if (perPage !== undefined) { + url += `&per_page=${perPage}` + } + const response = await request(url) await ensureSuccess(response) - const payload = await response.json() as unknown - if (!Array.isArray(payload)) { - throw new Error('Unexpected response while loading task definitions.') - } + const json = await response.json() + const { items, pagination: paginationData } = await parse_list_response(json) - const summaries: Array = payload.map(item => { - if (!item || 'object' !== typeof item) { - throw new Error('Encountered malformed task definition entry.') - } - - const entry = item as Record - return { - id: String(entry.id ?? ''), - name: String(entry.name ?? ''), - priority: Number(entry.priority ?? 0), - updated_at: Number(entry.updated_at ?? 0), - } - }) - - definitions.value = sortSummaries(summaries) + definitions.value = sortSummaries(items) + pagination.value = paginationData lastError.value = null } catch (error) { @@ -158,29 +145,13 @@ const loadDefinitions = async (): Promise => { * @param id Task definition ID * @returns TaskDefinitionDetailed or null on error */ -const getDefinition = async (id: string): Promise => { +const getDefinition = async (id: number): Promise => { try { - const response = await request(`/api/task_definitions/${id}`) + const response = await request(`/api/tasks/definitions/${id}`) await ensureSuccess(response) - const payload = await response.json() as unknown - if (!payload || 'object' !== typeof payload) { - throw new Error('Unexpected response while retrieving task definition.') - } - - const entry = payload as Record - if (!('definition' in entry) || 'object' !== typeof entry.definition) { - throw new Error('Task definition response is missing definition payload.') - } - - const detailed: TaskDefinitionDetailed = { - id: String(entry.id ?? ''), - name: String(entry.name ?? ''), - priority: Number(entry.priority ?? 0), - updated_at: Number(entry.updated_at ?? 0), - definition: entry.definition as TaskDefinitionDocument, - } - + const payload = await response.json() + const detailed = await parse_api_response(payload) lastError.value = null return detailed } @@ -198,19 +169,22 @@ const getDefinition = async (id: string): Promise */ const createDefinition = async (definition: TaskDefinitionDocument): Promise => { try { - const response = await request('/api/task_definitions/', { + const response = await request('/api/tasks/definitions/', { method: 'POST', - body: JSON.stringify({ definition }), + body: JSON.stringify(definition), }) await ensureSuccess(response) - const payload = await response.json() as TaskDefinitionDetailed + const payload = await parse_api_response(response.json()) updateSummaries({ id: payload.id, name: payload.name, priority: payload.priority, + match_url: payload.match_url, + enabled: payload.enabled, + created_at: payload.created_at, updated_at: payload.updated_at, }) @@ -231,21 +205,24 @@ const createDefinition = async (definition: TaskDefinitionDocument): Promise => { +const updateDefinition = async (id: number, definition: TaskDefinitionDocument): Promise => { try { - const response = await request(`/api/task_definitions/${id}`, { + const response = await request(`/api/tasks/definitions/${id}`, { method: 'PUT', - body: JSON.stringify({ definition }), + body: JSON.stringify(definition), }) await ensureSuccess(response) - const payload = await response.json() as TaskDefinitionDetailed + const payload = await parse_api_response(response.json()) updateSummaries({ id: payload.id, name: payload.name, priority: payload.priority, + match_url: payload.match_url, + enabled: payload.enabled, + created_at: payload.created_at, updated_at: payload.updated_at, }) @@ -265,9 +242,9 @@ const updateDefinition = async (id: string, definition: TaskDefinitionDocument): * @param id Task definition ID * @returns true if deleted, false on error */ -const deleteDefinition = async (id: string): Promise => { +const deleteDefinition = async (id: number): Promise => { try { - const response = await request(`/api/task_definitions/${id}`, { method: 'DELETE' }) + const response = await request(`/api/tasks/definitions/${id}`, { method: 'DELETE' }) await ensureSuccess(response) removeSummary(id) @@ -282,6 +259,44 @@ const deleteDefinition = async (id: string): Promise => { } } +/** + * Toggles the enabled status of a task definition. + * @param id Task definition ID + * @param enabled New enabled status + * @returns Updated TaskDefinitionDetailed or null on error + */ +const toggleEnabled = async (id: number, enabled: boolean): Promise => { + try { + const response = await request(`/api/tasks/definitions/${id}`, { + method: 'PATCH', + body: JSON.stringify({ enabled }), + }) + + await ensureSuccess(response) + + const payload = await parse_api_response(response.json()) + + updateSummaries({ + id: payload.id, + name: payload.name, + priority: payload.priority, + match_url: payload.match_url, + enabled: payload.enabled, + created_at: payload.created_at, + updated_at: payload.updated_at, + }) + + notify.success(`Task definition ${enabled ? 'enabled' : 'disabled'}.`) + lastError.value = null + return payload + } + catch (error) { + handleError(error) + if (throwInstead.value) throw error + return null + } +} + /** * Clears the last error message. */ @@ -295,6 +310,7 @@ const clearError = () => lastError.value = null */ export const useTaskDefinitions = () => ({ definitions: readonly(definitions), + pagination: readonly(pagination), isLoading: readonly(isLoading), lastError: readonly(lastError), loadDefinitions, @@ -302,6 +318,7 @@ export const useTaskDefinitions = () => ({ createDefinition, updateDefinition, deleteDefinition, + toggleEnabled, clearError, throwInstead, }) diff --git a/ui/app/pages/task_definitions.vue b/ui/app/pages/task_definitions.vue index 7c8bb197..b35dd11d 100644 --- a/ui/app/pages/task_definitions.vue +++ b/ui/app/pages/task_definitions.vue @@ -75,14 +75,23 @@ - - {{ definition.name || '(Unnamed definition)' }} + +
{{ definition.name || '(Unnamed definition)' }}
+
+ + + + + {{ definition.enabled ? 'Enabled' : 'Disabled' }} + +
{{ definition.priority }} - @@ -122,9 +131,19 @@ {{ definition.name || '(Unnamed definition)' }}
- +
+
+ + + +
+
+ +
+
@@ -139,8 +158,8 @@ Updated: @@ -219,14 +238,17 @@ import type { const DEFAULT_DEFINITION: TaskDefinitionDocument = { name: 'New Definition', priority: 0, - match: ['https://example.com/*'], - parse: { - items: { - type: 'css', - selector: 'body', - fields: { - link: { type: 'css', expression: 'a', attribute: 'href' }, - title: { type: 'css', expression: 'a', attribute: 'text' }, + enabled: true, + match_url: ['https://example.com/*'], + definition: { + parse: { + items: { + type: 'css', + selector: 'body', + fields: { + link: { type: 'css', expression: 'a', attribute: 'href' }, + title: { type: 'css', expression: 'a', attribute: 'text' }, + }, }, }, }, @@ -242,6 +264,7 @@ const getDefinition = taskDefs.getDefinition const createDefinition = taskDefs.createDefinition const updateDefinition = taskDefs.updateDefinition const deleteDefinition = taskDefs.deleteDefinition +const toggleEnabled = taskDefs.toggleEnabled const definitions = computed(() => definitionsRef.value) @@ -253,7 +276,7 @@ const editorMode = ref<'create' | 'edit'>('create') const editorLoading = ref(false) const editorSubmitting = ref(false) const workingDefinition = ref(null) -const workingId = ref(null) +const workingId = ref(null) const inspect = ref(false) const display_style = useStorage<'list' | 'grid'>('task-definitions:display', 'grid') @@ -299,39 +322,37 @@ const openEdit = async (summary: TaskDefinitionSummary): Promise => { return } - const document = cloneDocument(detailed.definition) - const docRecord = document - if ('priority' in docRecord) { - docRecord.priority = Number(docRecord.priority) - } - else { - docRecord.priority = detailed.priority + const document: TaskDefinitionDocument = { + name: detailed.name, + priority: detailed.priority, + enabled: detailed.enabled, + match_url: [...detailed.match_url], + definition: JSON.parse(JSON.stringify(detailed.definition)), } workingDefinition.value = document editorLoading.value = false } -const importExistingDefinition = async (id: string): Promise => { +const importExistingDefinition = async (id: number): Promise => { const detailed = await getDefinition(id) if (!detailed) { toast.error('Failed to load task definition for import.') return } - const document = cloneDocument(detailed.definition) - const docRecord = document - if ('priority' in docRecord) { - docRecord.priority = Number(docRecord.priority) - } - else { - docRecord.priority = detailed.priority + const document: TaskDefinitionDocument = { + name: detailed.name, + priority: detailed.priority, + enabled: detailed.enabled, + match_url: [...detailed.match_url], + definition: JSON.parse(JSON.stringify(detailed.definition)), } + editorMode.value = 'create' + workingId.value = null workingDefinition.value = document - if ('create' === editorMode.value) { - workingId.value = null - } + isEditorOpen.value = true editorLoading.value = false } @@ -374,7 +395,7 @@ const submitDefinition = async (definition: TaskDefinitionDocument): Promise => { const result = await confirmDialog({ title: 'Delete Task Definition', - message: `Are you sure you want to delete “${summary.name || summary.id}”?`, + message: `Are you sure you want to delete "${summary.name || summary.id}"?`, confirmColor: 'is-danger', }) @@ -385,21 +406,25 @@ const remove = async (summary: TaskDefinitionSummary): Promise => { await deleteDefinition(summary.id) } +const toggle = async (summary: TaskDefinitionSummary): Promise => { + await toggleEnabled(summary.id, !summary.enabled) +} + const exportDefinition = async (summary: TaskDefinitionSummary): Promise => { const detailed = await getDefinition(summary.id) if (!detailed) { return } - const payload = { + return copyText(encode({ _type: 'task_definition', - _version: '1.0', + _version: '2.0', name: detailed.name, priority: detailed.priority, + enabled: detailed.enabled, + match_url: detailed.match_url, definition: detailed.definition, - } - - return copyText(encode(payload)) + })) } onMounted(async () => { diff --git a/ui/app/types/task_definitions.ts b/ui/app/types/task_definitions.ts index 0c1f0583..0ecfb563 100644 --- a/ui/app/types/task_definitions.ts +++ b/ui/app/types/task_definitions.ts @@ -1,6 +1,5 @@ // --- Task Definition Schema Types --- - -export type TaskMatchPattern = | string | { regex?: string; glob?: string } +import type { Paginated } from '~/types/responses' export type EngineType = 'httpx' | 'selenium' @@ -29,7 +28,7 @@ export interface RequestConfig { headers?: StringMap params?: StringMap data?: StringMap | string | null - json?: object | Array | string | number | boolean | null + json_data?: object | Array | string | number | boolean | null timeout?: number } @@ -79,31 +78,37 @@ export interface ParseConfig { [field: string]: ExtractionRule | Container | undefined } -export interface TaskDefinitionDocument { - name: string - match: Array +export interface TaskDefinitionConfig { parse: ParseConfig - priority?: number engine?: EngineConfig request?: RequestConfig response?: ResponseConfig } -// --- Summaries and Error Types --- +export interface TaskDefinitionDocument { + name: string + match_url: string[] + priority?: number + enabled?: boolean + definition: TaskDefinitionConfig +} export type TaskDefinitionSummary = { - id: string, - name: string, - priority: number, - updated_at: number, + id: number + name: string + priority: number + match_url: ReadonlyArray + enabled: boolean + created_at: string + updated_at: string } export type TaskDefinitionDetailed = TaskDefinitionSummary & { - definition: TaskDefinitionDocument + definition: TaskDefinitionConfig } +export type TaskDefinitionList = Paginated + export type TaskDefinitionErrorResponse = { error: string, } - - diff --git a/ui/tests/composables/useTaskDefinitions.test.ts b/ui/tests/composables/useTaskDefinitions.test.ts index 881540f2..66ae514c 100644 --- a/ui/tests/composables/useTaskDefinitions.test.ts +++ b/ui/tests/composables/useTaskDefinitions.test.ts @@ -1,27 +1,37 @@ import * as utils from '~/utils/index' -import { describe, it, expect, beforeEach, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' + import { useTaskDefinitions } from '~/composables/useTaskDefinitions' -import type { TaskDefinitionSummary } from '~/types/task_definitions' +import type { TaskDefinitionDetailed, TaskDefinitionSummary } from '~/types/task_definitions' vi.mock('~/composables/useNotification', () => { const success = vi.fn() const error = vi.fn() return { useNotification: () => ({ success, error }), - default: () => ({ success, error }) + default: () => ({ success, error }), } }) -// Sample data const summary: TaskDefinitionSummary = { - id: 'abc', + id: 1, name: 'Test', priority: 1, updated_at: 123456, } +const listPayload = (items: TaskDefinitionSummary[]) => ({ + items, + pagination: { + page: 1, + per_page: 50, + total: items.length, + total_pages: 1, + has_next: false, + has_prev: false, + }, +}) -// Helper to create a mock Response object function createMockResponse({ ok, status, jsonData }: { ok: boolean, status: number, jsonData: any }) { return { ok, @@ -43,32 +53,31 @@ function createMockResponse({ ok, status, jsonData }: { ok: boolean, status: num } describe('useTaskDefinitions', () => { - beforeEach(() => { vi.clearAllMocks() }) it('sorts definitions by priority then name', async () => { const items = [ - { id: '1', name: 'B', priority: 2, updated_at: 1 }, - { id: '2', name: 'A', priority: 2, updated_at: 2 }, - { id: '3', name: 'C', priority: 1, updated_at: 3 }, + { id: 1, name: 'B', priority: 2, updated_at: 1 }, + { id: 2, name: 'A', priority: 2, updated_at: 2 }, + { id: 3, name: 'C', priority: 1, updated_at: 3 }, ] vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({ ok: true, status: 200, - jsonData: items, + jsonData: listPayload(items), })) const defs = useTaskDefinitions() await defs.loadDefinitions() - expect(defs.definitions.value.map(d => d.id)).toEqual(['3', '2', '1']) + expect(defs.definitions.value.map(d => d.id)).toEqual([3, 2, 1]) }) it('handles empty payload', async () => { vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({ ok: true, status: 200, - jsonData: [], + jsonData: listPayload([]), })) const defs = useTaskDefinitions() await defs.loadDefinitions() @@ -76,115 +85,66 @@ describe('useTaskDefinitions', () => { expect(defs.lastError.value).toBeNull() }) - it('handles malformed payload', async () => { - vi.spyOn(utils, 'request').mockResolvedValueOnce({ - ok: true, - json: async () => ({ jsonData: [{}] }) - }) - const defs = useTaskDefinitions() - await expect(defs.loadDefinitions()).resolves.toBeUndefined() - }) - - it('handles malformed payload (throws when throwInstead is true)', async () => { - vi.spyOn(utils, 'request').mockResolvedValueOnce({ - ok: true, - json: async () => ({ jsonData: [{}] }) - }) + it('throws loadDefinitions error when throwInstead is true', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({ + ok: false, + status: 500, + jsonData: { error: 'Server error' }, + })) const defs = useTaskDefinitions() defs.throwInstead.value = true - await expect(defs.loadDefinitions()).rejects.toThrow() + await expect(defs.loadDefinitions()).rejects.toThrow('Server error') }) - it('handles duplicate IDs (no deduplication, both present)', async () => { - vi.spyOn(utils, 'request').mockResolvedValueOnce({ - ok: true, - json: async () => [ - { id: 'dup', name: 'A', priority: 1 }, - { id: 'dup', name: 'B', priority: 2 }, - ] - }) - const defs = useTaskDefinitions() - await defs.loadDefinitions() - expect(defs.definitions.value.length).toBe(2) - expect(defs.definitions.value[0].name).toBe('A') - expect(defs.definitions.value[1].name).toBe('B') - }) - - it('handles error on getDefinition', async () => { - vi.spyOn(utils, 'request').mockResolvedValueOnce({ + it('returns null on getDefinition error', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({ ok: false, status: 404, - statusText: 'Not Found', - json: async () => ({}) - }) + jsonData: { error: 'Not Found' }, + })) const defs = useTaskDefinitions() - await expect(defs.getDefinition('notfound')).rejects.toThrow('Request failed with status 404') + defs.throwInstead.value = false // Reset from previous test + const result = await defs.getDefinition(123) + expect(result).toBeNull() + expect(defs.lastError.value).toBe('Not Found') }) - it('handles malformed getDefinition response', async () => { - vi.spyOn(utils, 'request').mockResolvedValueOnce({ - ok: true, - json: async () => ({}) - }) - const defs = useTaskDefinitions() - await expect(defs.getDefinition('bad')).rejects.toThrow('Task definition response is missing definition payload.') - }) - - it('handles error on createDefinition', async () => { - vi.spyOn(utils, 'request').mockResolvedValueOnce({ - ok: false, - status: 400, - statusText: 'Bad Request', - json: async () => ({}) - }) - const defs = useTaskDefinitions() - await expect(defs.createDefinition({ id: 'fail', name: 'Fail', priority: 1 })).rejects.toThrow('Request failed with status 400') - }) - - it('handles error on updateDefinition', async () => { - vi.spyOn(utils, 'request').mockResolvedValueOnce({ - ok: false, - status: 400, - statusText: 'Bad Request', - json: async () => ({}) - }) - const defs = useTaskDefinitions() - await expect(defs.updateDefinition({ id: 'fail', name: 'Fail', priority: 1 })).rejects.toThrow('Request failed with status 400') - }) - - it('handles error on deleteDefinition', async () => { - vi.spyOn(utils, 'request').mockResolvedValueOnce({ - ok: false, - status: 400, - statusText: 'Bad Request', - json: async () => ({}) - }) - const defs = useTaskDefinitions() - await expect(defs.deleteDefinition('fail')).rejects.toThrow('Request failed with status 400') - }) - - it('loads definitions successfully (duplicate test)', async () => { + it('calls success notification on createDefinition', async () => { + const payload: TaskDefinitionDetailed = { + id: 2, + name: 'New', + priority: 0, + updated_at: 999, + definition: { name: 'New', match: ['https://example.com'], parse: { link: { type: 'css', expression: 'a' } } }, + } vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({ ok: true, status: 200, - jsonData: [summary], + jsonData: payload, + })) + const defs = useTaskDefinitions() + await defs.createDefinition(payload.definition) + const notify = (await import('~/composables/useNotification')).useNotification() + expect(notify.success).toHaveBeenCalledWith('Task definition created.') + expect(defs.definitions.value.some(item => item.id === payload.id)).toBe(true) + }) + + it('removes definition on deleteDefinition', async () => { + vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({ + ok: true, + status: 200, + jsonData: listPayload([summary]), })) const defs = useTaskDefinitions() await defs.loadDefinitions() - expect(defs.definitions.value).toEqual([summary]) - expect(defs.lastError.value).toBeNull() - }) - - it('calls success notification on createDefinition', async () => { - vi.spyOn(utils, 'request').mockResolvedValueOnce({ + vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({ ok: true, - json: async () => ({}) - }) - const defs = useTaskDefinitions() - await defs.createDefinition({ id: 'new', name: 'New', priority: 1 }) - // Access the spy directly from the mock - const notify = (await import('~/composables/useNotification')).useNotification() - expect(notify.success).toHaveBeenCalledWith('Task definition created.') + status: 200, + jsonData: { status: 'deleted' }, + })) + const result = await defs.deleteDefinition(summary.id) + expect(result).toBe(true) + expect(defs.definitions.value).toEqual([]) }) -}); +})