danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

refactor: migrate task definitions to db model

Browse source
This commit is contained in:
arabcoders 2026-01-23 18:54:42 +03:00
parent e551ad7848
commit 042379035a
46 changed files with 2564 additions and 2077 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

280
API.md
View file

@ -42,11 +42,12 @@ This document describes the available endpoints and their usage. All endpoints r
- [DELETE /api/tasks/{id}/mark](#delete-apitasksidmark)
- [POST /api/tasks/{id}/metadata](#post-apitasksidmetadata)
- [PATCH /api/tasks/{id}](#patch-apitasksid)
- [GET /api/task\_definitions/](#get-apitask_definitions)
- [GET /api/task\_definitions/{identifier}](#get-apitask_definitionsidentifier)
- [POST /api/task\_definitions/](#post-apitask_definitions)
- [PUT /api/task\_definitions/{identifier}](#put-apitask_definitionsidentifier)
- [DELETE /api/task\_definitions/{identifier}](#delete-apitask_definitionsidentifier)
- [GET /api/tasks/definitions/](#get-apitasksdefinitions)
- [GET /api/tasks/definitions/{id}](#get-apitasksdefinitionsid)
- [POST /api/tasks/definitions/](#post-apitasksdefinitions)
- [PUT /api/tasks/definitions/{id}](#put-apitasksdefinitionsid)
- [PATCH /api/tasks/definitions/{id}](#patch-apitasksdefinitionsid)
- [DELETE /api/tasks/definitions/{id}](#delete-apitasksdefinitionsid)
- [GET /api/player/playlist/{file:.\*}.m3u8](#get-apiplayerplaylistfilem3u8)
- [GET /api/player/m3u8/{mode}/{file:.\*}.m3u8](#get-apiplayerm3u8modefilem3u8)
- [GET /api/player/segments/{segment}/{file:.\*}.ts](#get-apiplayersegmentssegmentfilets)
@ -70,11 +71,11 @@ This document describes the available endpoints and their usage. All endpoints r
- [PUT /api/presets](#put-apipresets)
- [GET /api/conditions/](#get-apiconditions)
- [POST /api/conditions/](#post-apiconditions)
- [POST /api/conditions/test](#post-apiconditionstest)
- [GET /api/conditions/{id}](#get-apiconditionsid)
- [PATCH /api/conditions/{id}](#patch-apiconditionsid)
- [PUT /api/conditions/{id}](#put-apiconditionsid)
- [DELETE /api/conditions/{id}](#delete-apiconditionsid)
- [POST /api/conditions/test](#post-apiconditionstest)
- [GET /api/logs](#get-apilogs)
- [GET /api/logs/stream](#get-apilogsstream)
- [GET /api/notifications/](#get-apinotifications)
@ -1140,54 +1141,71 @@ Returns the updated task
---
### GET /api/task_definitions/
**Purpose**: Retrieve all task definitions.
### GET /api/tasks/definitions/
**Purpose**: Retrieve task definitions.
**Query Parameters**:
- `include=definition` (optional) - Include the full definition object in response.
**Response**:
```json
[
{
"id": "<uuid>",
"name": "Task Definition Name",
"description": "...",
"enabled": true,
"definition": { ... } // only if include=definition
},
...
]
```
---
### GET /api/task_definitions/{identifier}
**Purpose**: Retrieve a specific task definition by ID or name.
**Path Parameter**:
- `identifier`: Task definition ID or name.
- `page` (optional): Page number (1-indexed). Default: `1`.
- `per_page` (optional): Items per page. Default: `config.default_pagination`.
**Response**:
```json
{
"id": "<uuid>",
"name": "Task Definition Name",
"description": "...",
"enabled": true,
"definition": {
"handler": "GenericTaskHandler",
"config": { ... }
"items": [
{
"id": 1,
"name": "Task Definition Name",
"description": "...",
"enabled": true,
"definition": { ... } // only if include=definition
},
...
],
"pagination": {
"page": 1,
"per_page": 50,
"total": 1,
"total_pages": 1,
"has_next": false,
"has_prev": false
}
}
```
- `400 Bad Request` if identifier is missing.
---
### GET /api/tasks/definitions/{id}
**Purpose**: Retrieve a specific task definition by ID.
**Path Parameter**:
- `id`: Task definition ID.
**Response**:
```json
{
"id": 1,
"name": "Task Definition Name",
"description": "...",
"enabled": true,
"definition": {
"parse": {
"url": { ... },
"items": { ... }
},
"engine": { ... },
"request": { ... },
"response": { ... }
}
}
```
- `400 Bad Request` if ID is missing.
- `404 Not Found` if the task definition doesn't exist.
---
### POST /api/task_definitions/
### POST /api/tasks/definitions/
**Purpose**: Create a new task definition.
**Body**:
@ -1197,19 +1215,13 @@ Returns the updated task
"description": "...",
"enabled": true,
"definition": {
"handler": "GenericTaskHandler",
"config": { ... }
}
}
```
Or wrap in a definition object:
```json
{
"definition": {
"name": "My Task Definition",
"handler": "GenericTaskHandler",
...
"parse": {
"url": { ... },
"items": { ... }
},
"engine": { ... },
"request": { ... },
"response": { ... }
}
}
```
@ -1217,7 +1229,7 @@ Or wrap in a definition object:
**Response**:
```json
{
"id": "<uuid>",
"id": 1,
"name": "My Task Definition",
"description": "...",
"enabled": true,
@ -1230,11 +1242,11 @@ Or wrap in a definition object:
---
### PUT /api/task_definitions/{identifier}
**Purpose**: Update an existing task definition.
### PUT /api/tasks/definitions/{id}
**Purpose**: Replace an existing task definition.
**Path Parameter**:
- `identifier`: Task definition ID or name.
- `id`: Task definition ID.
**Body**:
```json
@ -1243,8 +1255,10 @@ Or wrap in a definition object:
"description": "...",
"enabled": false,
"definition": {
"handler": "GenericTaskHandler",
"config": { ... }
"parse": { ... },
"engine": { ... },
"request": { ... },
"response": { ... }
}
}
```
@ -1252,7 +1266,7 @@ Or wrap in a definition object:
**Response**:
```json
{
"id": "<uuid>",
"id": 1,
"name": "Updated Name",
"description": "...",
"enabled": false,
@ -1261,23 +1275,53 @@ Or wrap in a definition object:
```
- `200 OK` if successful.
- `400 Bad Request` if identifier is missing or validation fails.
- `400 Bad Request` if ID is missing or validation fails.
- `404 Not Found` if the task definition doesn't exist.
---
### DELETE /api/task_definitions/{identifier}
### PATCH /api/tasks/definitions/{id}
**Purpose**: Partially update a task definition.
**Path Parameter**:
- `id`: Task definition ID.
**Body**:
```json
{
"enabled": false,
"description": "Updated description"
}
```
**Response**: Updated task definition object.
- `200 OK` if successful.
- `400 Bad Request` if ID is missing or validation fails.
- `404 Not Found` if the task definition doesn't exist.
---
### DELETE /api/tasks/definitions/{id}
**Purpose**: Delete a task definition.
**Path Parameter**:
- `identifier`: Task definition ID or name.
- `id`: Task definition ID.
**Response**:
```json
{ "status": "deleted" }
{
"id": 1,
"name": "Deleted Definition",
"description": "...",
"enabled": false,
"definition": { ... }
}
```
- `200 OK` if successful.
- `400 Bad Request` if identifier is missing or task definition doesn't exist.
- `400 Bad Request` if ID is missing.
- `404 Not Found` if the task definition doesn't exist.
---
@ -1616,6 +1660,9 @@ Binary image data with appropriate headers
### GET /api/dl_fields/{id}
**Purpose**: Retrieve a single download field by ID.
**Path Parameter**:
- `id`: Download field ID.
**Response**:
```json
{ "id": 1, "name": "Title", "description": "...", "field": "title", "kind": "text", "order": 0, "value": "", "icon": "fa-solid fa-tag", "extras": {} }
@ -1626,6 +1673,9 @@ Binary image data with appropriate headers
### PATCH /api/dl_fields/{id}
**Purpose**: Partially update a download field.
**Path Parameter**:
- `id`: Download field ID.
**Body**:
```json
{ "description": "Updated description", "order": 1 }
@ -1638,6 +1688,9 @@ Binary image data with appropriate headers
### PUT /api/dl_fields/{id}
**Purpose**: Replace a download field.
**Path Parameter**:
- `id`: Download field ID.
**Body**:
```json
{
@ -1659,6 +1712,9 @@ Binary image data with appropriate headers
### DELETE /api/dl_fields/{id}
**Purpose**: Delete a download field by ID.
**Path Parameter**:
- `id`: Download field ID.
**Response**:
```json
{ "id": 1, "name": "Title", "description": "...", "field": "title", "kind": "text", "order": 0, "value": "", "icon": "fa-solid fa-tag", "extras": {} }
@ -1759,41 +1815,6 @@ Binary image data with appropriate headers
---
### GET /api/conditions/{id}
**Purpose**: Retrieve a condition by ID.
**Response**: Condition object.
---
### PATCH /api/conditions/{id}
**Purpose**: Partially update a condition.
**Body**:
```json
{ "enabled": false, "priority": 5 }
```
**Response**: Updated condition object.
---
### PUT /api/conditions/{id}
**Purpose**: Replace a condition.
**Body**: Full condition object.
**Response**: Updated condition object.
---
### DELETE /api/conditions/{id}
**Purpose**: Delete a condition by ID.
**Response**: Deleted condition object.
---
### POST /api/conditions/test
**Purpose**: Evaluate a condition expression against info extracted from a URL.
@ -1815,6 +1836,53 @@ Binary image data with appropriate headers
---
### GET /api/conditions/{id}
**Purpose**: Retrieve a condition by ID.
**Path Parameter**:
- `id`: Condition ID.
**Response**: Condition object.
---
### PATCH /api/conditions/{id}
**Purpose**: Partially update a condition.
**Path Parameter**:
- `id`: Condition ID.
**Body**:
```json
{ "enabled": false, "priority": 5 }
```
**Response**: Updated condition object.
---
### PUT /api/conditions/{id}
**Purpose**: Replace a condition.
**Path Parameter**:
- `id`: Condition ID.
**Body**: Full condition object.
**Response**: Updated condition object.
---
### DELETE /api/conditions/{id}
**Purpose**: Delete a condition by ID.
**Path Parameter**:
- `id`: Condition ID.
**Response**: Deleted condition object.
---
### GET /api/logs
**Purpose**: Retrieve recent application logs (if file logging is enabled).
@ -1946,6 +2014,9 @@ Binary image data with appropriate headers
### GET /api/notifications/{id}
**Purpose**: Retrieve a notification target by ID.
**Path Parameter**:
- `id`: Notification target ID.
**Response**: Notification target object.
---
@ -1953,6 +2024,9 @@ Binary image data with appropriate headers
### PATCH /api/notifications/{id}
**Purpose**: Partially update a notification target.
**Path Parameter**:
- `id`: Notification target ID.
**Body**:
```json
{ "enabled": false }
@ -1965,6 +2039,9 @@ Binary image data with appropriate headers
### PUT /api/notifications/{id}
**Purpose**: Replace a notification target.
**Path Parameter**:
- `id`: Notification target ID.
**Body**: Full notification target object.
**Response**: Updated notification target.
@ -1974,6 +2051,9 @@ Binary image data with appropriate headers
### DELETE /api/notifications/{id}
**Purpose**: Delete a notification target.
**Path Parameter**:
- `id`: Notification target ID.
**Response**: Deleted notification target.
---
@ -2186,7 +2266,7 @@ The WebSocket API provides real-time bidirectional communication between the cli
### Connection
**URL**: `ws://localhost:8081/ws` (development) or `wss://yourdomain.com/ws` (production)
**URL**: `ws://localhost:8081/ws` (development) or `wss://domain.example/ws` (production)
The client automatically connects to the WebSocket server and receives a `connected` event with initial state. The frontend wrapper handles reconnection (default: up to 50 attempts, 5s delay).

26
FAQ.md
View file

@ -171,25 +171,25 @@ Then restart the container to apply the changes.
# How can I monitor sites without RSS feeds?
YTPTube includes a **generic task handler** that turns JSON definition files into site-specific scrapers. You can use it
YTPTube includes a **generic task handler** that turns JSON definitions into site-specific scrapers. You can use it
to watch pages that do not expose RSS or public APIs and automatically enqueue new links into the download queue.
1. Create definition files under `/config/tasks/*.json` (for Docker this is the mounted `config/tasks/` folder).
2. Keep your scheduled task in `tasks.json` pointing at the page you want to monitor and make sure it uses a preset that
enables a download archive (`--download-archive`).
3. When the task runs, the handler scans the JSON files, picks the first definition whose `match` rule covers the task
URL, fetches the page, extracts items, and queues the unseen ones.
1. Create definition via the WebUI > tasks > Definitions.
2. Create task that reference same url click on inspect to see the results. Make sure it uses a preset that enables
a download archive (`--download-archive`).
3. When the task handler run, the handler scans the definitions, picks the first definition whose `match` rule covers
the task URL, fetches the page, extracts items, and queues the unseen ones.
### Definition schema
Each file must contain a single JSON object with the following keys:
Each definition must contain a single JSON object with the following keys:
```json5
{
"name": "example", // Friendly identifier shown in logs
"match": [
"https://example.com/articles/*", // Glob strings, or objects with {"regex": "..."} or {"glob": "..."}
{ "regex": "https://example.com/post/[0-9]+" }
"match_url": [
"https://example.com/articles/*", // Glob strings
"https://example.com/post/[0-9]+" // Regex strings
],
"engine": { // Optional, defaults to HTTPX
"type": "httpx", // "httpx" (default) or "selenium"
@ -207,7 +207,7 @@ Each file must contain a single JSON object with the following keys:
"headers": { "User-Agent": "MyAgent/1.0" },
"params": { "page": 1 },
"data": null,
"json": null,
"json_data": null,
"timeout": 30
},
"response": { // Optional: how to interpret the body
@ -247,6 +247,7 @@ For JSON endpoints, switch the response format and use `jsonpath` selectors:
```json5
{
...
"response": { "type": "json" },
"parse": {
"items": {
@ -282,9 +283,6 @@ For JSON endpoints, switch the response format and use `jsonpath` selectors:
the moment. Optional keys: `arguments` (list or string), `wait_for` (type `css`/`xpath` + `expression`), `wait_timeout`,
and `page_load_timeout`.
Definitions are reloaded automatically when files change, so you can tweak them without restarting YTPTube. Check
`var/config/tasks/01-*.json` for sample files.
> [!NOTE]
> A machine-readable schema is available at `app/schema/task_definition.json` if you want to validate your JSON with editors or CI tools.

11
app/features/conditions/repository.py
View file

@ -120,13 +120,13 @@ class ConditionsRepository(metaclass=Singleton):
clause: ColumnElement[bool] = ConditionModel.name == identifier
result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1))
model: ConditionModel | None = result.scalar_one_or_none()
if not model:
if not (model := result.scalar_one_or_none()):
msg: str = f"Condition '{identifier}' not found."
raise KeyError(msg)
payload.pop("id", None)
payload.pop("created_at", None)
payload.pop("updated_at", None)
if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None:
msg: str = f"Condition with name '{payload['name']}' already exists."
@ -142,7 +142,6 @@ class ConditionsRepository(metaclass=Singleton):
async def delete(self, identifier: int | str) -> ConditionModel:
async with self.session() as session:
# Query within this session
if isinstance(identifier, int):
clause: ColumnElement[bool] = ConditionModel.id == identifier
elif isinstance(identifier, str) and identifier.isdigit():
@ -153,9 +152,7 @@ class ConditionsRepository(metaclass=Singleton):
clause: ColumnElement[bool] = ConditionModel.name == identifier
result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1))
model = result.scalar_one_or_none()
if not model:
if not (model := result.scalar_one_or_none()):
msg: str = f"Condition '{identifier}' not found."
raise KeyError(msg)

8
app/features/conditions/router.py
View file

@ -177,7 +177,7 @@ async def conditions_add(request: Request, encoder: Encoder, notify: EventBus) -
return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode)
@route("GET", "api/conditions/{id}", name="condition_get")
@route("GET", r"api/conditions/{id:\d+}", name="condition_get")
async def conditions_get(request: Request, encoder: Encoder) -> Response:
"""
Get the conditions
@ -199,7 +199,7 @@ async def conditions_get(request: Request, encoder: Encoder) -> Response:
return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode)
@route("DELETE", "api/conditions/{id}", name="condition_delete")
@route("DELETE", r"api/conditions/{id:\d+}", name="condition_delete")
async def conditions_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response:
"""
Delete Condition.
@ -226,7 +226,7 @@ async def conditions_delete(request: Request, encoder: Encoder, notify: EventBus
return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code)
@route("PATCH", "api/conditions/{id}", name="condition_patch")
@route("PATCH", r"api/conditions/{id:\d+}", name="condition_patch")
async def conditions_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response:
"""
Patch Condition.
@ -277,7 +277,7 @@ async def conditions_patch(request: Request, encoder: Encoder, notify: EventBus)
return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode)
@route("PUT", "api/conditions/{id}", name="condition_update")
@route("PUT", r"api/conditions/{id:\d+}", name="condition_update")
async def conditions_update(request: Request, encoder: Encoder, notify: EventBus) -> Response:
"""
Update Condition.

3
app/features/core/migration.py
View file

@ -2,7 +2,6 @@ from __future__ import annotations
import abc
import logging
import shutil
import time
from pathlib import Path
from typing import TYPE_CHECKING
@ -46,7 +45,7 @@ class Migration(abc.ABC):
timestamp = int(time.time())
destination: Path = self._migrated_dir / f"{source.stem}_{timestamp}{source.suffix}"
shutil.move(str(source), str(destination))
source.rename(destination)
return destination
def _unique_name(self, name: str, seen_names: dict[str, int]) -> str:

1
app/features/core/schemas.py
View file

@ -20,6 +20,7 @@ class CEFeature(str, Enum):
DL_FIELDS = "dl_fields"
CONDITIONS = "conditions"
NOTIFICATIONS = "notifications"
TASKS_DEFINITIONS = "tasks_definitions"
def __str__(self) -> str:
return self.value

8
app/features/dl_fields/router.py
View file

@ -67,7 +67,7 @@ async def dl_fields_add(request: Request, encoder: Encoder, notify: EventBus) ->
return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode)
@route("GET", "api/dl_fields/{id}", "dl_fields_get")
@route("GET", r"api/dl_fields/{id:\d+}", "dl_fields_get")
async def dl_fields_get(request: Request, encoder: Encoder) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -78,7 +78,7 @@ async def dl_fields_get(request: Request, encoder: Encoder) -> Response:
return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode)
@route("DELETE", "api/dl_fields/{id}", "dl_fields_delete")
@route("DELETE", r"api/dl_fields/{id:\d+}", "dl_fields_delete")
async def dl_fields_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -97,7 +97,7 @@ async def dl_fields_delete(request: Request, encoder: Encoder, notify: EventBus)
return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code)
@route("PATCH", "api/dl_fields/{id}", "dl_fields_patch")
@route("PATCH", r"api/dl_fields/{id:\d+}", "dl_fields_patch")
async def dl_fields_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -138,7 +138,7 @@ async def dl_fields_patch(request: Request, encoder: Encoder, notify: EventBus)
)
@route("PUT", "api/dl_fields/{id}", "dl_fields_update")
@route("PUT", r"api/dl_fields/{id:\d+}", "dl_fields_update")
async def dl_fields_update(request: Request, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)

8
app/features/notifications/router.py
View file

@ -83,7 +83,7 @@ async def notifications_add(request: Request, encoder: Encoder, notify: EventBus
return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode)
@route("GET", "api/notifications/{id}", "notification_get")
@route("GET", r"api/notifications/{id:\d+}", "notification_get")
async def notifications_get(request: Request, encoder: Encoder) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -94,7 +94,7 @@ async def notifications_get(request: Request, encoder: Encoder) -> Response:
return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode)
@route("DELETE", "api/notifications/{id}", "notification_delete")
@route("DELETE", r"api/notifications/{id:\d+}", "notification_delete")
async def notifications_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -114,7 +114,7 @@ async def notifications_delete(request: Request, encoder: Encoder, notify: Event
return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code)
@route("PATCH", "api/notifications/{id}", "notification_patch")
@route("PATCH", r"api/notifications/{id:\d+}", "notification_patch")
async def notifications_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -180,7 +180,7 @@ async def notifications_patch(request: Request, encoder: Encoder, notify: EventB
)
@route("PUT", "api/notifications/{id}", "notification_update")
@route("PUT", r"api/notifications/{id:\d+}", "notification_update")
async def notifications_update(request: Request, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)

1
app/features/tasks/__init__.py Normal file
View file

@ -0,0 +1 @@
"""Tasks Feature."""

1
app/features/tasks/definitions/__init__.py Normal file
View file

@ -0,0 +1 @@
"""Tasks Definitions Feature."""

5
app/features/tasks/definitions/deps.py Normal file
View file

@ -0,0 +1,5 @@
from app.features.tasks.definitions.repository import TaskDefinitionsRepository
def get_task_definitions_repo() -> TaskDefinitionsRepository:
return TaskDefinitionsRepository.get_instance()

1
app/features/tasks/definitions/handlers/__init__.py Normal file
View file

@ -0,0 +1 @@
"""Handlers package for task definitions."""

2
app/library/task_handlers/_base_handler.py → app/features/tasks/definitions/handlers/_base_handler.py
View file

@ -11,7 +11,7 @@ from app.library.Tasks import Task, TaskFailure, TaskResult
class BaseHandler:
@staticmethod
def can_handle(task: Task) -> bool:
async def can_handle(task: Task) -> bool:
return False
@staticmethod

753
app/library/task_handlers/generic.py → app/features/tasks/definitions/handlers/generic.py
View file

File diff suppressed because it is too large Load diff

2
app/library/task_handlers/rss.py → app/features/tasks/definitions/handlers/rss.py
View file

@ -24,7 +24,7 @@ class RssGenericHandler(BaseHandler):
)
@staticmethod
def can_handle(task: Task) -> bool:
async def can_handle(task: Task) -> bool:
LOG.debug(f"'{task.name}': Checking if task URL is parsable RSS feed: {task.url}")
return RssGenericHandler.parse(task.url) is not None

8
app/library/task_handlers/tver.py → app/features/tasks/definitions/handlers/tver.py
View file

@ -10,9 +10,9 @@ LOG: logging.Logger = logging.getLogger(__name__)
class TverHandler(BaseHandler):
SERIES_API = "https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/{id}"
SESSION_API = "https://platform-api.tver.jp/v2/api/platform_users/browser/create"
HEADERS = {
SERIES_API: str = "https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/{id}"
SESSION_API: str = "https://platform-api.tver.jp/v2/api/platform_users/browser/create"
HEADERS: dict[str, str] = {
"x-tver-platform-type": "web",
"Origin": "https://tver.jp",
"Referer": "https://tver.jp/",
@ -21,7 +21,7 @@ class TverHandler(BaseHandler):
RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?tver\.jp\/series\/(?P<id>sr[a-z0-9_]+)$")
@staticmethod
def can_handle(task: Task) -> bool:
async def can_handle(task: Task) -> bool:
LOG.debug(f"Checking if task '{task.name}' is using parsable Tver series URL: {task.url}")
return TverHandler.parse(task.url) is not None

4
app/library/task_handlers/twitch.py → app/features/tasks/definitions/handlers/twitch.py
View file

@ -15,12 +15,12 @@ LOG: logging.Logger = logging.getLogger(__name__)
class TwitchHandler(BaseHandler):
FEED = "https://twitchrss.appspot.com/vodonly/{handle}"
FEED: str = "https://twitchrss.appspot.com/vodonly/{handle}"
RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?twitch\.tv\/(?P<id>[a-z0-9_]{3,25})(?:\/.*)?$")
@staticmethod
def can_handle(task: Task) -> bool:
async def can_handle(task: Task) -> bool:
LOG.debug(f"Checking if task '{task.name}' is using parsable Twitch URL: {task.url}")
return TwitchHandler.parse(task.url) is not None

4
app/library/task_handlers/youtube.py → app/features/tasks/definitions/handlers/youtube.py
View file

@ -15,7 +15,7 @@ LOG: logging.Logger = logging.getLogger(__name__)
class YoutubeHandler(BaseHandler):
FEED = "https://www.youtube.com/feeds/videos.xml?{type}={id}"
FEED: str = "https://www.youtube.com/feeds/videos.xml?{type}={id}"
CHANNEL_REGEX: re.Pattern[str] = re.compile(
r"^https?://(?:www\.)?youtube\.com/(?:channel/(?P<id>UC[0-9A-Za-z_-]{22})|)/?$"
@ -26,7 +26,7 @@ class YoutubeHandler(BaseHandler):
)
@staticmethod
def can_handle(task: Task) -> bool:
async def can_handle(task: Task) -> bool:
LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}")
return YoutubeHandler.parse(task.url) is not None

126
app/features/tasks/definitions/migration.py Normal file
View file

@ -0,0 +1,126 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any
from app.features.core.migration import Migration as FeatureMigration
from app.library.config import Config
if TYPE_CHECKING:
from app.features.tasks.definitions.repository import TaskDefinitionsRepository
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
name: str = "task_definitions"
def __init__(self, repo: TaskDefinitionsRepository, config: Config | None = None) -> None:
self._config: Config = config or Config.get_instance()
super().__init__(config=self._config)
self._repo: TaskDefinitionsRepository = repo
self._source_dir: Path = Path(self._config.config_path) / "tasks"
async def should_run(self) -> bool:
if not self._source_dir.exists():
return False
return any(self._source_dir.glob("*.json"))
async def migrate(self) -> None:
if await self._repo.count() > 0:
LOG.warning("Task definitions already exist in the database; skipping migration.")
await self._archive_sources()
return
inserted = 0
seen_names: dict[str, int] = {}
for path in sorted(self._source_dir.glob("*.json")):
normalized = await self._normalize(path, seen_names)
if not normalized:
await self._move_file(path)
continue
try:
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception("Failed to insert task definition '%s': %s", normalized.get("name"), exc)
finally:
await self._move_file(path)
LOG.info("Migrated %s task definition(s) from %s.", inserted, self._source_dir)
async def _archive_sources(self) -> None:
for path in self._source_dir.glob("*.json"):
await self._move_file(path)
async def _normalize(self, path: Path, seen_names: dict[str, int]) -> dict[str, Any] | None:
try:
content = path.read_text(encoding="utf-8")
except Exception as exc:
LOG.error("Failed to read task definition '%s': %s", path, exc)
return None
try:
payload = json.loads(content)
except Exception as exc:
LOG.error("Failed to parse JSON for '%s': %s", path, exc)
return None
if not isinstance(payload, dict):
LOG.error("Task definition in '%s' must be a JSON object.", path)
return None
if "match" in payload and "match_url" not in payload:
payload["match_url"] = payload.pop("match")
# Normalize match_url from old object format to new string format
if "match_url" in payload and isinstance(payload["match_url"], list):
normalized_match: list[str] = []
for item in payload["match_url"]:
if isinstance(item, str):
normalized_match.append(item)
elif isinstance(item, dict):
if "regex" in item and isinstance(item["regex"], str):
# Convert {regex: "pattern"} to /pattern/
normalized_match.append(f"/{item['regex']}/")
elif "glob" in item and isinstance(item["glob"], str):
# Convert {glob: "pattern"} to pattern
normalized_match.append(item["glob"])
payload["match_url"] = normalized_match
# Rename request.json to request.json_data
if "request" in payload and isinstance(payload["request"], dict) and "json" in payload["request"]:
payload["request"]["json_data"] = payload["request"].pop("json")
if "definition" not in payload:
definition_fields = {}
for field in ["parse", "engine", "request", "response"]:
if field in payload:
definition_fields[field] = payload.pop(field)
if definition_fields:
payload["definition"] = definition_fields
# Also handle nested definition.request.json
elif isinstance(payload["definition"], dict):
if (
"request" in payload["definition"]
and isinstance(payload["definition"]["request"], dict)
and "json" in payload["definition"]["request"]
):
payload["definition"]["request"]["json_data"] = payload["definition"]["request"].pop("json")
name_value = payload.get("name")
if not isinstance(name_value, str) or not name_value.strip():
LOG.error("Task definition in '%s' missing a valid name.", path)
return None
name = self._unique_name(name_value.strip(), seen_names)
payload["name"] = name
# Repository will handle validation and field extraction
return payload

27
app/features/tasks/definitions/models.py Normal file
View file

@ -0,0 +1,27 @@
from __future__ import annotations
from datetime import datetime # noqa: TC003
from sqlalchemy import JSON, Boolean, Index, Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from app.features.core.models import Base, UTCDateTime, utcnow
class TaskDefinitionModel(Base):
__tablename__: str = "task_definitions"
__table_args__: tuple[Index, ...] = (
Index("ix_task_definitions_name", "name"),
Index("ix_task_definitions_priority", "priority"),
Index("ix_task_definitions_match_url", "match_url"),
Index("ix_task_definitions_enabled", "enabled"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
match_url: Mapped[list] = mapped_column(JSON, nullable=False)
definition: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False)
updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False)

184
app/features/tasks/definitions/repository.py Normal file
View file

@ -0,0 +1,184 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import func, or_, select
from app.features.core.deps import get_session
from app.features.core.schemas import CEFeature, ConfigEvent
from app.features.tasks.definitions.migration import Migration
from app.features.tasks.definitions.models import TaskDefinitionModel
from app.library.Events import Event, EventBus, Events
from app.library.Services import Services
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
LOG: logging.Logger = logging.getLogger(__name__)
class TaskDefinitionsRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
return
self._migrated = True
await Migration(repo=self).run()
@staticmethod
def get_instance() -> TaskDefinitionsRepository:
return TaskDefinitionsRepository()
def attach(self, _: Any) -> None:
async def handle_event(_, __):
await self.run_migrations()
async def handler(e: Event, __):
from app.features.tasks.definitions.handlers.generic import GenericTaskHandler
if isinstance(e.data, ConfigEvent) and CEFeature.TASKS_DEFINITIONS == e.data.feature:
LOG.debug("Refreshing task definitions due to configuration update.")
await GenericTaskHandler.refresh_definitions(force=True)
Services.get_instance().add(__class__.__name__, self)
EventBus.get_instance().subscribe(
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions")
async def list(self) -> list[TaskDefinitionModel]:
async with self.session() as session:
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
)
return list(result.scalars().all())
async def list_paginated(self, page: int, per_page: int) -> tuple[list[TaskDefinitionModel], int, int, int]:
async with self.session() as session:
total: int = await self.count()
total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1
if page > total_pages and total > 0:
page = total_pages
query: Select[tuple[TaskDefinitionModel]] = (
select(TaskDefinitionModel)
.order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
.limit(per_page)
.offset((page - 1) * per_page)
)
result: Result[tuple[TaskDefinitionModel]] = await session.execute(query)
return list(result.scalars().all()), total, page, total_pages
async def count(self) -> int:
async with self.session() as session:
result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(TaskDefinitionModel))
return int(result.scalar_one())
async def get(self, identifier: int | str) -> TaskDefinitionModel | None:
async with self.session() as session:
if not identifier:
return None
if isinstance(identifier, int):
clause: ColumnElement[bool] = TaskDefinitionModel.id == identifier
elif isinstance(identifier, str) and identifier.isdigit():
clause = or_(TaskDefinitionModel.id == int(identifier), TaskDefinitionModel.name == identifier)
else:
clause = TaskDefinitionModel.name == identifier
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
select(TaskDefinitionModel).where(clause).limit(1)
)
return result.scalar_one_or_none()
async def get_by_name(self, name: str, exclude_id: int | None = None) -> TaskDefinitionModel | None:
async with self.session() as session:
query: Select[tuple[TaskDefinitionModel]] = select(TaskDefinitionModel).where(
TaskDefinitionModel.name == name
)
if exclude_id is not None:
query = query.where(TaskDefinitionModel.id != exclude_id)
result: Result[tuple[TaskDefinitionModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel:
async with self.session() as session:
model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Task definition with name '{model.name}' already exists."
raise ValueError(msg)
session.add(model)
await session.commit()
await session.refresh(model)
return model
async def update(self, identifier: int | str, payload: dict[str, Any]) -> TaskDefinitionModel:
async with self.session() as session:
if isinstance(identifier, int):
clause: ColumnElement[bool] = TaskDefinitionModel.id == identifier
elif isinstance(identifier, str) and identifier.isdigit():
clause = or_(TaskDefinitionModel.id == int(identifier), TaskDefinitionModel.name == identifier)
else:
clause = TaskDefinitionModel.name == identifier
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
select(TaskDefinitionModel).where(clause).limit(1)
)
model: TaskDefinitionModel | None = result.scalar_one_or_none()
if not model:
msg: str = f"Task definition '{identifier}' not found."
raise KeyError(msg)
payload.pop("id", None)
payload.pop("created_at", None)
payload.pop("updated_at", None)
if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None:
msg = f"Task definition with name '{payload['name']}' already exists."
raise ValueError(msg)
for key, value in payload.items():
if hasattr(model, key):
setattr(model, key, value)
await session.commit()
await session.refresh(model)
return model
async def delete(self, identifier: int | str) -> TaskDefinitionModel:
async with self.session() as session:
if isinstance(identifier, int):
clause: ColumnElement[bool] = TaskDefinitionModel.id == identifier
elif isinstance(identifier, str) and identifier.isdigit():
clause = or_(TaskDefinitionModel.id == int(identifier), TaskDefinitionModel.name == identifier)
else:
clause = TaskDefinitionModel.name == identifier
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
select(TaskDefinitionModel).where(clause).limit(1)
)
if not (model := result.scalar_one_or_none()):
msg: str = f"Task definition '{identifier}' not found."
raise KeyError(msg)
await session.delete(model)
await session.commit()
return model

235
app/features/tasks/definitions/router.py Normal file
View file

@ -0,0 +1,235 @@
import logging
from typing import Any
from aiohttp import web
from aiohttp.web import Request, Response
from pydantic import ValidationError
from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination
from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination
from app.features.tasks.definitions.repository import TaskDefinitionsRepository as Repo
from app.features.tasks.definitions.schemas import (
TaskDefinition,
TaskDefinitionList,
TaskDefinitionPatch,
)
from app.features.tasks.definitions.utils import model_to_schema, schema_to_payload
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.router import route
LOG: logging.Logger = logging.getLogger(__name__)
@route("GET", "api/tasks/definitions/", "task_definitions")
async def task_definitions_list(request: Request, encoder: Encoder, repo: Repo) -> Response:
page, per_page = normalize_pagination(request)
models, total, current_page, total_pages = await repo.list_paginated(page, per_page)
include: str | None = request.query.get("include")
summary: bool = "definition" != include
return web.json_response(
data=TaskDefinitionList(
items=[model_to_schema(model, summary=summary) for model in models],
pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)),
),
status=web.HTTPOk.status_code,
dumps=encoder.encode,
)
@route("GET", r"api/tasks/definitions/{id:\d+}", "task_definitions_get")
async def task_definitions_get(request: Request, encoder: Encoder, repo: Repo) -> Response:
identifier: str = request.match_info.get("id", "").strip()
if not identifier:
return web.json_response(
data={"error": "Missing task definition identifier."},
status=web.HTTPBadRequest.status_code,
)
if not (model := await repo.get(identifier)):
return web.json_response(
data={"error": f"Task definition '{identifier}' not found."},
status=web.HTTPNotFound.status_code,
)
definition = model_to_schema(model)
return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode)
@route("POST", "api/tasks/definitions/", "task_definitions_create")
async def task_definitions_create(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response:
try:
payload: Any = await request.json()
except Exception:
return web.json_response(
data={"error": "Invalid JSON in request body."},
status=web.HTTPBadRequest.status_code,
)
if not isinstance(payload, dict):
return web.json_response(
data={"error": "Invalid request body; expected JSON object."},
status=web.HTTPBadRequest.status_code,
)
try:
definition_input = TaskDefinition.model_validate(payload)
except ValidationError as exc:
return web.json_response(
data={"error": "Failed to validate task definition.", "detail": format_validation_errors(exc)},
status=web.HTTPBadRequest.status_code,
)
try:
repo_payload = schema_to_payload(definition_input)
model = await repo.create(repo_payload)
definition = model_to_schema(model)
notify.emit(
Events.CONFIG_UPDATE,
data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.CREATE, data=definition.model_dump()),
)
return web.json_response(data=definition, status=web.HTTPCreated.status_code, dumps=encoder.encode)
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
except Exception as exc:
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to create task definition."},
status=web.HTTPInternalServerError.status_code,
)
@route("PUT", r"api/tasks/definitions/{id:\d+}", "task_definitions_update")
async def task_definitions_update(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response:
if not (identifier := request.match_info.get("id", "").strip()):
return web.json_response(
data={"error": "Missing task definition identifier."},
status=web.HTTPBadRequest.status_code,
)
try:
payload: dict | None = await request.json()
except Exception:
return web.json_response(
data={"error": "Invalid JSON in request body."},
status=web.HTTPBadRequest.status_code,
)
if not isinstance(payload, dict):
return web.json_response(
data={"error": "Invalid request body; expected JSON object."},
status=web.HTTPBadRequest.status_code,
)
try:
definition_input: TaskDefinition = TaskDefinition.model_validate(payload)
except ValidationError as exc:
return web.json_response(
data={"error": "Failed to validate task definition.", "detail": format_validation_errors(exc)},
status=web.HTTPBadRequest.status_code,
)
try:
definition: TaskDefinition = model_to_schema(await repo.update(identifier, schema_to_payload(definition_input)))
notify.emit(
Events.CONFIG_UPDATE,
data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.UPDATE, data=definition.model_dump()),
)
return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode)
except KeyError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code)
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
except Exception as exc:
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to update task definition."},
status=web.HTTPInternalServerError.status_code,
)
@route("PATCH", r"api/tasks/definitions/{id:\d+}", "task_definitions_patch")
async def task_definitions_patch(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response:
if not (identifier := request.match_info.get("id", "").strip()):
return web.json_response(
data={"error": "Missing task definition identifier."},
status=web.HTTPBadRequest.status_code,
)
if not await repo.get(identifier):
return web.json_response(
data={"error": f"Task definition '{identifier}' not found."},
status=web.HTTPNotFound.status_code,
)
try:
payload: dict | None = await request.json()
except Exception:
return web.json_response(
data={"error": "Invalid JSON in request body."},
status=web.HTTPBadRequest.status_code,
)
if not isinstance(payload, dict):
return web.json_response(
data={"error": "Invalid request body; expected JSON object."},
status=web.HTTPBadRequest.status_code,
)
try:
patch_input: TaskDefinitionPatch = TaskDefinitionPatch.model_validate(payload)
except ValidationError as exc:
return web.json_response(
data={"error": "Failed to validate task definition patch.", "detail": format_validation_errors(exc)},
status=web.HTTPBadRequest.status_code,
)
try:
definition: TaskDefinition = model_to_schema(
await repo.update(identifier, patch_input.model_dump(exclude_unset=True))
)
notify.emit(
Events.CONFIG_UPDATE,
data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.UPDATE, data=definition.model_dump()),
)
return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode)
except KeyError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code)
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
except Exception as exc:
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to patch task definition."},
status=web.HTTPInternalServerError.status_code,
)
@route("DELETE", r"api/tasks/definitions/{id:\d+}", "task_definitions_delete")
async def task_definitions_delete(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response:
if not (identifier := request.match_info.get("id", "").strip()):
return web.json_response(
data={"error": "Missing task definition identifier."},
status=web.HTTPBadRequest.status_code,
)
try:
definition: TaskDefinition = model_to_schema(await repo.delete(identifier))
notify.emit(
Events.CONFIG_UPDATE,
data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.DELETE, data=definition.model_dump()),
)
return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode)
except KeyError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code)
except Exception as exc:
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to delete task definition."},
status=web.HTTPInternalServerError.status_code,
)

222
app/features/tasks/definitions/schemas.py Normal file
View file

@ -0,0 +1,222 @@
from __future__ import annotations
import re
from datetime import datetime # noqa: TC003
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from app.features.core.schemas import Pagination
from app.features.core.utils import parse_int
class PostFilter(BaseModel):
model_config = ConfigDict(str_strip_whitespace=True)
filter: str = Field(min_length=1)
value: str | None = None
@field_validator("filter")
@classmethod
def _validate_filter(cls, value: str) -> str:
try:
re.compile(value)
except re.error as exc:
msg: str = f"Invalid post_filter regex pattern: {exc}"
raise ValueError(msg) from exc
return value
class ExtractionRule(BaseModel):
model_config = ConfigDict(str_strip_whitespace=True)
type: Literal["css", "xpath", "regex", "jsonpath"]
expression: str = Field(min_length=1)
attribute: str | None = None
post_filter: PostFilter | None = None
def __getitem__(self, key: str) -> Any:
"""Support bracket notation access."""
if not hasattr(self, key):
raise KeyError(key)
return getattr(self, key)
class ParseItems(BaseModel):
model_config = ConfigDict(str_strip_whitespace=True)
type: Literal["css", "xpath", "jsonpath"] = "css"
selector: str | None = Field(None, min_length=1)
expression: str | None = Field(None, min_length=1)
fields: dict[str, ExtractionRule]
def get(self, key: str, default: Any = None) -> Any:
"""Get a field value by key, supporting dict-like access."""
return getattr(self, key, default)
def __getitem__(self, key: str) -> Any:
"""Support bracket notation access."""
if not hasattr(self, key):
raise KeyError(key)
return getattr(self, key)
@model_validator(mode="after")
def _validate_items(self) -> ParseItems:
if not self.selector and not self.expression:
msg = "Either 'selector' or 'expression' must be provided."
raise ValueError(msg)
if not self.selector:
self.selector = self.expression
if "link" not in self.fields:
msg = "Container 'fields' must include a 'link' field."
raise ValueError(msg)
return self
class Parse(BaseModel):
model_config = ConfigDict(str_strip_whitespace=True, extra="allow")
items: ParseItems | None = None
def get(self, key: str, default: Any = None) -> Any:
"""Get a field value by key, supporting dict-like access."""
return getattr(self, key, default)
def field_items(self) -> list[tuple[str, Any]]:
"""Return field items like a dict, excluding private fields and 'items'."""
data: dict[str, Any] = self.model_dump()
return [(k, v) for k, v in data.items() if k not in ("items",)]
def __getitem__(self, key: str) -> Any:
"""Support bracket notation access."""
if not hasattr(self, key):
raise KeyError(key)
return getattr(self, key)
def __contains__(self, key: str) -> bool:
"""Support 'in' operator."""
return hasattr(self, key) and not key.startswith("_")
@model_validator(mode="before")
@classmethod
def _validate_parse(cls, value: Any) -> Any:
"""Validate that we have either items or direct parsers with link."""
if not isinstance(value, dict):
msg: str = "Parse must be a dict"
raise ValueError(msg)
has_items: bool = "items" in value and value["items"] is not None
direct_parsers: dict[str, Any] = {
k: v for k, v in value.items() if k not in ("items",) and not k.startswith("_")
}
has_direct_parsers: bool = len(direct_parsers) > 0
has_link_parser: bool = "link" in direct_parsers
if not has_items and not has_direct_parsers:
msg: str = "Field 'parse' must contain either 'items' or direct parsers."
raise ValueError(msg)
if not has_items and not has_link_parser:
msg: str = "Missing required 'link' parser definition."
raise ValueError(msg)
for field_name, field_value in direct_parsers.items():
if not isinstance(field_value, dict):
msg: str = f"Parse field '{field_name}' must be an object."
raise ValueError(msg)
return value
class EngineConfig(BaseModel):
model_config = ConfigDict(str_strip_whitespace=True)
type: Literal["httpx", "selenium"] = "httpx"
options: dict[str, Any] = Field(default_factory=dict)
class RequestConfig(BaseModel):
model_config = ConfigDict(str_strip_whitespace=True, protected_namespaces=())
method: str = "GET"
headers: dict[str, str] = Field(default_factory=dict)
params: dict[str, Any] = Field(default_factory=dict)
data: dict[str, Any] | None = None
json_data: dict[str, Any] | None = None
timeout: float | None = None
url: str | None = None
class ResponseConfig(BaseModel):
model_config = ConfigDict(str_strip_whitespace=True)
type: Literal["html", "json"] = "html"
class Definition(BaseModel):
model_config = ConfigDict(str_strip_whitespace=True)
parse: Parse
engine: EngineConfig = Field(default_factory=EngineConfig)
request: RequestConfig = Field(default_factory=RequestConfig)
response: ResponseConfig = Field(default_factory=ResponseConfig)
class TaskDefinitionSummary(BaseModel):
model_config = ConfigDict(from_attributes=True, str_strip_whitespace=True)
id: int | None = None
name: str = Field(min_length=1)
priority: int = Field(default=0, ge=0)
match_url: list[str] = Field(min_length=1)
enabled: bool = Field(default=True)
created_at: datetime | None = None
updated_at: datetime | None = None
class TaskDefinition(TaskDefinitionSummary):
definition: Definition
@field_validator("priority", mode="before")
@classmethod
def _normalize_priority(cls, value: Any) -> int:
if value is None:
return 0
return parse_int(value, field="Priority", minimum=0)
@field_validator("match_url", mode="before")
@classmethod
def _validate_match_url(cls, value: Any) -> list[str]:
"""Validate that match_url is a list of strings and validate regex patterns."""
if not isinstance(value, list):
msg = "match_url must be a list"
raise ValueError(msg)
validated: list[str] = []
for item in value:
if not isinstance(item, str):
msg: str = f"match_url items must be strings, got {type(item).__name__}"
raise ValueError(msg)
item: str = item.strip()
if not item:
msg = "match_url items cannot be empty"
raise ValueError(msg)
if item.startswith("/") and item.endswith("/") and len(item) > 2:
pattern = item[1:-1]
try:
re.compile(pattern)
except re.error as exc:
msg = f"Invalid regex pattern '{pattern}': {exc}"
raise ValueError(msg) from exc
validated.append(item)
return validated
class TaskDefinitionPatch(TaskDefinition):
model_config = ConfigDict(str_strip_whitespace=True)
name: str | None = None
priority: int | None = None
match_url: list[str] | None = None
definition: Definition | None = None
enabled: bool | None = None
class TaskDefinitionList(BaseModel):
model_config = ConfigDict(from_attributes=True)
items: list[TaskDefinitionSummary | TaskDefinition] = Field(default_factory=list)
pagination: Pagination

395
app/features/tasks/definitions/tests/test_generic_task_handler.py Normal file
View file

@ -0,0 +1,395 @@
from datetime import datetime
from unittest.mock import patch
import pytest
from app.features.tasks.definitions.handlers.generic import GenericTaskHandler
from app.features.tasks.definitions.schemas import (
EngineConfig,
RequestConfig,
ResponseConfig,
TaskDefinition,
Definition,
)
from app.library.Tasks import Task, TaskFailure, TaskResult
@pytest.fixture(autouse=True)
def reset_generic_handler(monkeypatch):
monkeypatch.setattr(GenericTaskHandler, "_definitions", [])
monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {})
def test_build_task_definition_parses_valid_payload():
definition = TaskDefinition(
id=1,
name="example",
priority=0,
match_url=["https://example.com/articles/*"],
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
),
)
assert definition is not None, "TaskDefinition should be created"
assert "example" == definition.name, "Name should match"
assert "https://example.com/articles/*" in definition.match_url, "Match URL should be in list"
assert "link" in definition.definition.parse, "Parse should contain link field"
assert ".article a.link::attr(href)" == definition.definition.parse["link"]["expression"], (
"Link expression should match"
)
def test_build_task_definition_handles_container():
definition = TaskDefinition(
id=2,
name="container",
priority=0,
match_url=["https://example.com/cards"],
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"selector": ".cards .card",
"fields": {
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
},
}
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
),
)
assert definition is not None, "TaskDefinition should be created"
assert "items" in definition.definition.parse, "Parse should contain items container"
assert ".cards .card" == definition.definition.parse["items"]["selector"], "Items selector should match"
assert "link" in definition.definition.parse["items"]["fields"], "Items fields should contain link"
def test_build_task_definition_handles_json():
definition = TaskDefinition(
id=3,
name="json-def",
priority=0,
match_url=["https://example.com/api"],
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
),
)
assert definition is not None, "TaskDefinition should be created"
assert "json" == definition.definition.response.type, "Response type should be json"
assert "items" in definition.definition.parse, "Parse should contain items container"
assert "jsonpath" == definition.definition.parse["items"]["type"], "Items type should be jsonpath"
assert "jsonpath" == definition.definition.parse["items"]["fields"]["link"]["type"], (
"Link field type should be jsonpath"
)
def test_parse_items_extracts_values():
definition = TaskDefinition(
id=4,
name="example",
priority=0,
match_url=["https://example.com/*"],
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
),
)
html = """
<div class="article" data-id="101">
<a class="link" href="/article-101">First</a>
<span class="title">First Title</span>
</div>
<div class="article" data-id="102">
<a class="link" href="https://example.com/article-102">Second</a>
<span class="title">Second Title</span>
</div>
"""
items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/")
assert 2 == len(items), "Should extract 2 items"
assert "https://example.com/article-101" == items[0]["link"], "First item link should be absolute URL"
assert "First Title" == items[0]["title"], "First item title should match"
assert "101" == items[0]["id"], "First item id should match"
assert "https://example.com/article-102" == items[1]["link"], "Second item link should match"
def test_parse_items_handles_nested_card_layout():
definition = TaskDefinition(
id=5,
name="nested",
priority=0,
match_url=["https://example.com/*"],
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "css",
"selector": ".columns .card",
"fields": {
"link": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "href",
},
"title": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "text",
},
"poet": {
"type": "css",
"expression": "footer .card-footer-item:first-child a",
"attribute": "text",
},
"category": {
"type": "css",
"expression": "footer .card-footer-item:nth-child(2) a",
"attribute": "text",
},
},
}
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
),
)
html = """
<div class="columns is-multiline">
<div class="column is-6">
<div class="card">
<div class="card-header">
<p class="card-header-title is-4 has-text-centered is-block text-truncate">
<a href="/poems/view/111" title="First Poem">First Poem</a>
</p>
</div>
<footer class="card-footer has-text-centered">
<p class="card-footer-item text-truncate">
<span class="text-truncate"> By <a href="/poet/alpha">Poet Alpha</a></span>
</p>
<p class="card-footer-item text-truncate">
<span class="text-truncate"> In <a href="/category/one">Category One</a></span>
</p>
</footer>
</div>
</div>
<div class="column is-6">
<div class="card">
<div class="card-header">
<p class="card-header-title is-4 has-text-centered is-block text-truncate">
<a href="/poems/view/222" title="Second Poem">Second Poem</a>
</p>
</div>
<footer class="card-footer has-text-centered">
<p class="card-footer-item text-truncate">
<span class="text-truncate"> By <a href="/poet/beta">Poet Beta</a></span>
</p>
</footer>
</div>
</div>
</div>
"""
items = GenericTaskHandler._parse_items(definition, html, "https://example.com")
assert 2 == len(items), "Should extract 2 items"
assert "https://example.com/poems/view/111" == items[0]["link"], "First item link should match"
assert "First Poem" == items[0]["title"], "First item title should match"
assert "Poet Alpha" == items[0]["poet"], "First item poet should match"
assert "Category One" == items[0]["category"], "First item category should match"
assert "https://example.com/poems/view/222" == items[1]["link"], "Second item link should match"
assert "Second Poem" == items[1]["title"], "Second item title should match"
assert "Poet Beta" == items[1]["poet"], "Second item poet should match"
assert "category" not in items[1], "Second item should not have category"
def test_parse_items_handles_json_container():
definition = TaskDefinition(
id=6,
name="json",
priority=0,
match_url=["https://example.com/*"],
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "jsonpath",
"selector": "entries",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
"id": {"type": "jsonpath", "expression": "id"},
},
}
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
),
)
payload = {
"entries": [
{"url": "/video/1", "title": "First", "id": 1},
{"url": "https://example.com/video/2", "title": "Second", "id": 2},
{"title": "Missing Link", "id": 3},
]
}
items = GenericTaskHandler._parse_items(
definition=definition,
html="",
base_url="https://example.com",
json_data=payload,
)
assert 2 == len(items), "Should extract 2 items (third missing link)"
assert "https://example.com/video/1" == items[0]["link"], "First item link should be absolute"
assert "First" == items[0]["title"], "First item title should match"
assert "1" == items[0]["id"], "First item id should match"
assert "https://example.com/video/2" == items[1]["link"], "Second item link should match"
assert "Second" == items[1]["title"], "Second item title should match"
assert "2" == items[1]["id"], "Second item id should match"
@pytest.mark.asyncio
async def test_generic_task_handler_inspect(monkeypatch):
definition = TaskDefinition(
id=7,
name="json-inspect",
priority=0,
match_url=["https://example.com/*"],
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
),
)
async def fake_find_definition(cls, url): # noqa: ARG001
return definition
monkeypatch.setattr(
GenericTaskHandler,
"_find_definition",
classmethod(fake_find_definition),
)
async def fake_fetch_content(url, definition, ytdlp_opts): # noqa: ARG001
return "", {"items": [{"url": "/video/1", "title": "First"}]}
monkeypatch.setattr(GenericTaskHandler, "_fetch_content", staticmethod(fake_fetch_content))
# Mock fetch_info to return valid info with required fields for archive ID generation
async def fake_fetch_info(config, url, **kwargs): # noqa: ARG001
return {"id": "test_video_1", "extractor_key": "Example"}
with patch("app.features.tasks.definitions.handlers.generic.fetch_info", side_effect=fake_fetch_info):
task = Task(id="inspect", name="Inspect", url="https://example.com/api")
result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task)
assert isinstance(result, TaskResult), "Result should be TaskResult"
assert 1 == len(result.items), "Should have 1 item"
item = result.items[0]
assert "https://example.com/video/1" == item.url, "Item URL should match"
assert "First" == item.title, "Item title should match"
def test_parse_items_handles_json_top_level_list():
definition = TaskDefinition(
id=8,
name="json-list",
priority=0,
match_url=["https://example.com/*"],
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "jsonpath",
"selector": "[]",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
),
)
payload = [
{"url": "/video/1", "title": "First"},
{"url": "/video/2", "title": "Second"},
]
items = GenericTaskHandler._parse_items(
definition=definition,
html="",
base_url="https://example.com",
json_data=payload,
)
assert 2 == len(items), "Should extract 2 items"
assert "https://example.com/video/1" == items[0]["link"], "First item link should match"
assert "First" == items[0]["title"], "First item title should match"
assert "https://example.com/video/2" == items[1]["link"], "Second item link should match"
assert "Second" == items[1]["title"], "Second item title should match"

6
app/tests/test_rss_handler.py → app/features/tasks/definitions/tests/test_rss_handler.py
View file

@ -1,6 +1,6 @@
import pytest
from app.library.task_handlers.rss import RssGenericHandler
from app.features.tasks.definitions.handlers.rss import RssGenericHandler
from app.library.Tasks import Task, TaskResult
@ -155,7 +155,7 @@ class TestRssHandlerExtraction:
preset="default",
)
assert RssGenericHandler.can_handle(task) is True
assert await RssGenericHandler.can_handle(task) is True
non_feed_task = Task(
id="test_youtube",
@ -164,7 +164,7 @@ class TestRssHandlerExtraction:
preset="default",
)
assert RssGenericHandler.can_handle(non_feed_task) is False
assert await RssGenericHandler.can_handle(non_feed_task) is False
class TestRssHandlerEdgeCases:

228
app/features/tasks/definitions/tests/test_task_definitions.py Normal file
View file

@ -0,0 +1,228 @@
from __future__ import annotations
import json
from collections.abc import AsyncGenerator
from unittest.mock import AsyncMock, MagicMock
import pytest
import pytest_asyncio
from aiohttp import web
from aiohttp.web import Request
from app.features.tasks.definitions.repository import TaskDefinitionsRepository
from app.features.tasks.definitions.router import (
task_definitions_create,
task_definitions_delete,
task_definitions_get,
task_definitions_list,
task_definitions_patch,
task_definitions_update,
)
from app.library.encoder import Encoder
from app.library.sqlite_store import SqliteStore
from app.main import EventBus
def _sample_definition(name: str = "example", *, priority: int = 0) -> dict:
"""Returns a properly structured task definition payload for the repository."""
return {
"name": name,
"match_url": ["https://example.com/*"],
"priority": priority,
"definition": {
"parse": {
"link": {
"type": "css",
"expression": "a",
"attribute": "href",
}
},
"engine": {"type": "httpx"},
"request": {},
"response": {"type": "html"},
},
}
@pytest_asyncio.fixture
async def repo() -> AsyncGenerator[TaskDefinitionsRepository, None]:
TaskDefinitionsRepository._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=":memory:")
await store.get_connection()
repository = TaskDefinitionsRepository.get_instance()
yield repository
if store._conn:
await store._conn.close()
if store._engine:
await store._engine.dispose()
TaskDefinitionsRepository._reset_singleton()
SqliteStore._reset_singleton()
class TestTaskDefinitionsRepository:
@pytest.mark.asyncio
async def test_create_and_list(self, repo: TaskDefinitionsRepository) -> None:
await repo.create(_sample_definition("Alpha", priority=2))
await repo.create(_sample_definition("Beta", priority=1))
items = await repo.list()
assert len(items) == 2, "Should return two task definitions"
assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name"
@pytest.mark.asyncio
async def test_create_duplicate_name_raises(self, repo: TaskDefinitionsRepository) -> None:
payload = _sample_definition("Dup")
await repo.create(payload)
with pytest.raises(ValueError, match="already exists"):
await repo.create(payload)
with pytest.raises(ValueError, match="already exists"):
await repo.create(payload)
@pytest.mark.asyncio
async def test_update_missing_raises(self, repo: TaskDefinitionsRepository) -> None:
with pytest.raises(KeyError, match="not found"):
await repo.update(999, {"name": "Missing"})
@pytest.mark.asyncio
async def test_list_paginated(self, repo: TaskDefinitionsRepository) -> None:
for idx in range(5):
await repo.create(_sample_definition(f"Item {idx}", priority=idx))
items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2)
assert len(items) == 2, "Should return two items per page"
assert total == 5, "Should report total count of 5"
assert page == 1, "Should return requested page"
assert total_pages == 3, "Should compute total pages"
@pytest.mark.asyncio
class TestTaskDefinitionRoutes:
async def test_list_definitions(self, repo: TaskDefinitionsRepository) -> None:
await repo.create(_sample_definition("Sample"))
request = MagicMock(spec=Request)
request.query = {}
response = await task_definitions_list(request, Encoder(), repo)
payload = json.loads(response.text)
assert response.status == web.HTTPOk.status_code, "Should return 200 for list"
assert payload["items"][0]["name"] == "Sample", "Should include created definition"
async def test_get_definition_not_found(self, repo: TaskDefinitionsRepository) -> None:
request = MagicMock(spec=Request)
request.match_info = {"id": "999"}
response = await task_definitions_get(request, Encoder(), repo)
payload = json.loads(response.text)
assert response.status == web.HTTPNotFound.status_code, "Should return 404 for missing definition"
assert "error" in payload, "Should include error payload"
async def test_create_definition_success(self, repo: TaskDefinitionsRepository) -> None:
request = MagicMock(spec=Request)
request.json = AsyncMock(return_value=_sample_definition("New", priority=3))
response = await task_definitions_create(request, Encoder(), MagicMock(spec=EventBus), repo)
body = json.loads(response.text)
assert response.status == web.HTTPCreated.status_code, "Should create task definition"
assert body["name"] == "New", "Should return created name"
assert body["priority"] == 3, "Should return created priority"
async def test_update_definition_success(self, repo: TaskDefinitionsRepository) -> None:
created = await repo.create(_sample_definition("Original", priority=0))
request = MagicMock(spec=Request)
request.match_info = {"id": str(created.id)}
request.json = AsyncMock(return_value=_sample_definition("Updated", priority=4))
response = await task_definitions_update(request, Encoder(), MagicMock(spec=EventBus), repo)
body = json.loads(response.text)
assert response.status == web.HTTPOk.status_code, "Should update task definition"
assert body["name"] == "Updated", "Should return updated name"
assert body["priority"] == 4, "Should return updated priority"
async def test_delete_definition_success(self, repo: TaskDefinitionsRepository) -> None:
created = await repo.create(_sample_definition("Delete"))
request = MagicMock(spec=Request)
request.match_info = {"id": str(created.id)}
response = await task_definitions_delete(request, Encoder(), MagicMock(spec=EventBus), repo)
assert response.status == web.HTTPOk.status_code, "Should delete task definition"
async def test_patch_definition_enabled(self, repo: TaskDefinitionsRepository) -> None:
created = await repo.create(_sample_definition("PatchTest", priority=5))
assert created.enabled is True, "Should be enabled by default"
request = MagicMock(spec=Request)
request.match_info = {"id": str(created.id)}
request.json = AsyncMock(return_value={"enabled": False})
response = await task_definitions_patch(request, Encoder(), MagicMock(spec=EventBus), repo)
body = json.loads(response.text)
assert response.status == web.HTTPOk.status_code, "Should patch task definition"
assert body["name"] == "PatchTest", "Should keep original name"
assert body["priority"] == 5, "Should keep original priority"
assert body["enabled"] is False, "Should update enabled status"
async def test_patch_definition_priority(self, repo: TaskDefinitionsRepository) -> None:
created = await repo.create(_sample_definition("PatchPriority", priority=1))
request = MagicMock(spec=Request)
request.match_info = {"id": str(created.id)}
request.json = AsyncMock(return_value={"priority": 10})
response = await task_definitions_patch(request, Encoder(), MagicMock(spec=EventBus), repo)
body = json.loads(response.text)
assert response.status == web.HTTPOk.status_code, "Should patch task definition"
assert body["priority"] == 10, "Should update priority"
assert body["enabled"] is True, "Should keep original enabled status"
async def test_patch_definition_not_found(self, repo: TaskDefinitionsRepository) -> None:
request = MagicMock(spec=Request)
request.match_info = {"id": "999"}
request.json = AsyncMock(return_value={"enabled": False})
response = await task_definitions_patch(request, Encoder(), MagicMock(spec=EventBus), repo)
payload = json.loads(response.text)
assert response.status == web.HTTPNotFound.status_code, "Should return 404 for missing definition"
assert "error" in payload, "Should include error payload"
async def test_create_with_regex_pattern(self, repo: TaskDefinitionsRepository) -> None:
payload = _sample_definition("RegexTest", priority=0)
payload["match_url"] = ["/https://example\\.com/post/[0-9]+/"]
request = MagicMock(spec=Request)
request.json = AsyncMock(return_value=payload)
response = await task_definitions_create(request, Encoder(), MagicMock(spec=EventBus), repo)
body = json.loads(response.text)
assert response.status == web.HTTPCreated.status_code, "Should create task definition with regex pattern"
assert body["match_url"][0] == "/https://example\\.com/post/[0-9]+/", "Should preserve regex pattern format"
async def test_create_with_invalid_regex_pattern(self, repo: TaskDefinitionsRepository) -> None:
payload = _sample_definition("BadRegex", priority=0)
payload["match_url"] = ["/[invalid(/"]
request = MagicMock(spec=Request)
request.json = AsyncMock(return_value=payload)
response = await task_definitions_create(request, Encoder(), MagicMock(spec=EventBus), repo)
payload_response = json.loads(response.text)
assert response.status == web.HTTPBadRequest.status_code, "Should reject invalid regex pattern"
assert "error" in payload_response, "Should include error payload"

9
app/tests/test_tver_handler.py → app/features/tasks/definitions/tests/test_tver_handler.py
View file

@ -1,6 +1,6 @@
import pytest
from app.library.task_handlers.tver import TverHandler
from app.features.tasks.definitions.handlers.tver import TverHandler
from app.library.Tasks import Task, TaskResult
@ -151,9 +151,10 @@ def test_tver_handler_parse(url: str, should_match: bool):
assert result is None
def test_tver_handler_can_handle():
@pytest.mark.asyncio
async def test_tver_handler_can_handle():
"""Test tver handler can_handle method."""
task_valid = Task(id="test1", name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
task_invalid = Task(id="test2", name="Test", url="https://youtube.com/watch?v=123", preset="default")
assert TverHandler.can_handle(task_valid) is True
assert TverHandler.can_handle(task_invalid) is False
assert await TverHandler.can_handle(task_valid) is True
assert await TverHandler.can_handle(task_invalid) is False

48
app/features/tasks/definitions/utils.py Normal file
View file

@ -0,0 +1,48 @@
from typing import Any
from app.features.tasks.definitions.models import TaskDefinitionModel
from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary
def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary:
"""
Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema.
Args:
model (TaskDefinitionModel): The model instance to convert.
summary (bool): Whether to return a summary schema.
Returns:
TaskDefinition | TaskDefinitionSummary: The corresponding schema instance.
"""
dct = {
"id": model.id,
"name": model.name,
"priority": model.priority,
"match_url": model.match_url,
"enabled": model.enabled,
"created_at": model.created_at,
"updated_at": model.updated_at,
}
return TaskDefinitionSummary(**dct) if summary else TaskDefinition(**dct, definition=Definition(**model.definition))
def schema_to_payload(item: TaskDefinition) -> dict[str, Any]:
"""
Convert a TaskDefinition schema to a dictionary payload for database operations.
Args:
item (TaskDefinition): The schema instance to convert.
Returns:
dict[str, Any]: The corresponding dictionary payload.
"""
return {
"name": item.name,
"priority": item.priority,
"match_url": item.match_url,
"enabled": item.enabled,
"definition": item.definition.model_dump(exclude_unset=True, exclude_none=True),
}

180
app/library/Services.py
View file

@ -1,6 +1,7 @@
import inspect
import logging
from typing import Any, TypeVar
from dataclasses import dataclass
from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints
from app.library.Singleton import Singleton
@ -8,59 +9,166 @@ T = TypeVar("T")
LOG: logging.Logger = logging.getLogger(__name__)
def _unwrap_annotation(ann: Any) -> Any:
if ann is inspect._empty:
return inspect._empty
origin = get_origin(ann)
# Annotated[T, ...] -> T
if origin is Annotated:
args = get_args(ann)
return args[0] if args else ann
# Optional[T] / Union[T, None] / T | None -> T
if origin is None:
return ann
if str(origin) in ("typing.Union", "types.UnionType"):
args = [a for a in get_args(ann) if a is not type(None)]
if len(args) == 1:
return args[0]
return ann
return ann
@dataclass(frozen=True)
class ServiceEntry:
name: str
declared_type: type | None
instance: Any
class Services(metaclass=Singleton):
def __init__(self):
self._services: dict[str, T] = {}
self._services: list[ServiceEntry] = []
@staticmethod
def get_instance() -> "Services":
return Services()
def add(self, name: str, service: T):
self._services[name] = service
def add(self, name: str, service: Any, declared_type: type | None = None):
"""
Add a service by name.
def add_all(self, services: dict[str, T]):
for name, service in services.items():
self.add(name, service)
Args:
name: The name of the service.
service: The service instance.
declared_type: The declared type of the service (optional).
def get(self, name: str) -> T | None:
return self._services.get(name)
"""
if declared_type is None and service is not None:
declared_type = type(service)
def has(self, name: str) -> bool:
return name in self._services
self.remove(name)
self._services.append(ServiceEntry(name=name, declared_type=declared_type, instance=service))
def add_all(self, services: dict[str, Any]):
for name, svc in services.items():
self.add(name, svc)
def remove(self, name: str):
if name not in self._services:
return
self._services.pop(name, None)
self._services = [e for e in self._services if e.name != name]
def clear(self):
self._services.clear()
def get_all(self) -> dict[str, T]:
def get(self, name: str) -> Any | None:
for e in reversed(self._services):
if e.name == name:
return e.instance
return None
def has(self, name: str) -> bool:
return any(e.name == name for e in self._services)
def get_all(self) -> list[ServiceEntry]:
return self._services.copy()
def get_by_type(self, expected_type: Any) -> Any | None:
expected_type = _unwrap_annotation(expected_type)
if expected_type is inspect._empty or not isinstance(expected_type, type):
return None
exact: list[Any] = [e.instance for e in self._services if e.declared_type is expected_type]
if len(exact) == 1:
return exact[0]
if len(exact) > 1:
msg: str = (
f"Ambiguous dependency for type {expected_type.__name__}: {len(exact)} exact matches. Resolve by name."
)
raise LookupError(msg)
candidates: list[Any] = [e.instance for e in self._services if isinstance(e.instance, expected_type)]
if len(candidates) == 0:
return None
if len(candidates) > 1:
msg: str = (
f"Ambiguous dependency for type {expected_type.__name__}: "
f"{len(candidates)} candidates. Resolve by name."
)
raise LookupError(msg)
return candidates[0]
def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]:
sig: inspect.Signature = inspect.signature(handler)
try:
type_hints: dict[str, Any] = get_type_hints(handler)
except Exception:
type_hints = {}
resolved: dict[str, Any] = {}
for name, param in sig.parameters.items():
if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):
continue
if name in overrides:
resolved[name] = overrides[name]
continue
by_name: Any | None = self.get(name)
if by_name is not None:
resolved[name] = by_name
continue
ann: Any | None = type_hints.get(name, param.annotation)
by_type: Any | None = self.get_by_type(ann)
if by_type is not None:
resolved[name] = by_type
continue
if param.default is not inspect._empty:
continue
missing_required: list[str] = []
for name, param in sig.parameters.items():
if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):
continue
if param.default is not inspect._empty:
continue
if name not in resolved:
missing_required.append(name)
if missing_required:
LOG.error(
"Missing arguments for handler '%s': %s",
getattr(handler, "__name__", str(handler)),
missing_required,
)
return resolved
async def handle_async(self, handler: callable, **kwargs) -> Any:
context = {**self.get_all(), **kwargs}
sig = inspect.signature(handler)
expected_args = sig.parameters.keys()
filtered = {k: v for k, v in context.items() if k in expected_args}
if missing_args := expected_args - filtered.keys():
LOG.error(f"Missing arguments for handler '{handler.__name__}': {missing_args}")
return await handler(**filtered)
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return await handler(**resolved)
def handle_sync(self, handler: callable, **kwargs) -> Any:
context = {**self.get_all(), **kwargs}
sig = inspect.signature(handler)
expected_args = sig.parameters.keys()
filtered = {k: v for k, v in context.items() if k in expected_args}
if missing_args := expected_args - filtered.keys():
LOG.error(f"Missing arguments for handler '{handler.__name__}': {missing_args}")
return handler(**filtered)
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return handler(**resolved)

18
app/library/Tasks.py
View file

@ -689,7 +689,7 @@ class HandleTask:
self._scheduler.add(
timer=timer,
func=self._dispatcher,
func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"),
id=f"{__class__.__name__}._dispatcher",
)
@ -704,7 +704,7 @@ class HandleTask:
if self._scheduler.has(self._task_name):
self._scheduler.remove(self._task_name)
def _dispatcher(self):
async def _dispatcher(self):
s: dict[list[str]] = {"h": [], "d": [], "u": [], "f": []}
handler_groups: dict[str, list[tuple[Task, type]]] = {}
@ -720,7 +720,7 @@ class HandleTask:
continue
try:
handler = self._find_handler(task)
handler = await self._find_handler(task)
if handler is None:
s["u"].append(task.name)
continue
@ -779,10 +779,10 @@ class HandleTask:
if exc := fut.exception():
LOG.error(f"Exception while handling task '{task.name}': {exc}")
def _find_handler(self, task: Task) -> type | None:
async def _find_handler(self, task: Task) -> type | None:
for cls in self._handlers:
try:
if Services.get_instance().handle_sync(handler=cls.can_handle, task=task):
if await Services.get_instance().handle_async(handler=cls.can_handle, task=task):
return cls
except Exception as e:
LOG.exception(e)
@ -804,7 +804,7 @@ class HandleTask:
"""
if not handler:
handler = self._find_handler(task)
handler = await self._find_handler(task)
if handler is None:
return None
@ -958,7 +958,7 @@ class HandleTask:
)
try:
matched = services.handle_sync(handler=handler_cls.can_handle, task=task)
matched = await services.handle_async(handler=handler_cls.can_handle, task=task)
except Exception as exc: # pragma: no cover - defensive
LOG.exception(exc)
message = str(exc)
@ -974,7 +974,7 @@ class HandleTask:
metadata={"matched": False, "handler": handler_cls.__name__},
)
else:
handler_cls = self._find_handler(task)
handler_cls = await self._find_handler(task)
if handler_cls is None:
message = "No handler matched the supplied URL."
return TaskFailure(
@ -1032,7 +1032,7 @@ class HandleTask:
def _discover(self) -> list[type]:
import importlib
import app.library.task_handlers as handlers_pkg
import app.features.tasks.definitions.handlers as handlers_pkg
handlers: list[type] = []

4
app/main.py
View file

@ -17,6 +17,7 @@ from aiohttp import web
from app.features.conditions.service import Conditions
from app.features.dl_fields.service import DLFields
from app.features.notifications.service import Notifications
from app.features.tasks.definitions.deps import get_task_definitions_repo
from app.library.BackgroundWorker import BackgroundWorker
from app.library.cache import Cache
from app.library.config import Config
@ -28,7 +29,6 @@ from app.library.Presets import Presets
from app.library.Scheduler import Scheduler
from app.library.Services import Services
from app.library.sqlite_store import SqliteStore
from app.library.TaskDefinitions import TaskDefinitions
from app.library.Tasks import Tasks
from app.library.UpdateChecker import UpdateChecker
@ -121,7 +121,7 @@ class Main:
Notifications.get_instance().attach(self._app)
Conditions.get_instance().attach(self._app)
DLFields.get_instance().attach(self._app)
TaskDefinitions.get_instance().attach(self._app)
get_task_definitions_repo().attach(self._app)
DownloadQueue.get_instance().attach(self._app)
UpdateChecker.get_instance().attach(self._app)

38
app/migrations/20260122201858_add_task_definitions.py Normal file
View file

@ -0,0 +1,38 @@
"""
This module contains a db migration.
Migration Name: add_task_definitions
Migration Version: 20260122201858
"""
from sqlalchemy import text
async def upgrade(c):
sql: list[str] = [
"""
CREATE TABLE IF NOT EXISTS "task_definitions" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"name" TEXT NOT NULL UNIQUE,
"priority" INTEGER NOT NULL DEFAULT 0,
"enabled" BOOLEAN NOT NULL DEFAULT 1,
"match_url" JSON NOT NULL,
"definition" JSON NOT NULL DEFAULT '{}',
"created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
""",
'CREATE INDEX IF NOT EXISTS "ix_task_definitions_name" ON "task_definitions" ("name");',
'CREATE INDEX IF NOT EXISTS "ix_task_definitions_priority" ON "task_definitions" ("priority");',
'CREATE INDEX IF NOT EXISTS "ix_task_definitions_match_url" ON "task_definitions" ("match_url");',
'CREATE INDEX IF NOT EXISTS "ix_task_definitions_enabled" ON "task_definitions" ("enabled");',
]
for sql_stmt in sql:
await c.execute(text(sql_stmt))
async def downgrade(c):
sql = """
DROP TABLE IF EXISTS "task_definitions";
"""
await c.execute(text(sql))

146
app/routes/api/task_definitions.py
View file

@ -1,145 +1,3 @@
import logging
from typing import Any
"""Migrated API routes for feature submodule."""
from aiohttp import web
from aiohttp.web import Request, Response
from app.library.encoder import Encoder
from app.library.router import route
from app.library.TaskDefinitions import TaskDefinitionRecord, TaskDefinitions
LOG: logging.Logger = logging.getLogger(__name__)
@route("GET", "api/task_definitions/", "task_definitions")
async def task_definitions_list(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response:
include: str | None = request.query.get("include")
include_definition: bool = "definition" == include
return web.json_response(
data=[item.serialize(include_definition=include_definition) for item in task_definitions.list()],
status=web.HTTPOk.status_code,
dumps=encoder.encode,
)
@route("GET", "api/task_definitions/{identifier}", "task_definitions_get")
async def task_definitions_get(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response:
identifier: str = request.match_info.get("identifier", "").strip()
if not identifier:
return web.json_response(
data={"error": "Missing task definition identifier."},
status=web.HTTPBadRequest.status_code,
)
record: TaskDefinitionRecord | None = task_definitions.get(identifier)
if not record:
return web.json_response(
data={"error": f"Task definition '{identifier}' not found."},
status=web.HTTPNotFound.status_code,
)
return web.json_response(
data=record.serialize(include_definition=True),
status=web.HTTPOk.status_code,
dumps=encoder.encode,
)
@route("POST", "api/task_definitions/", "task_definitions_create")
async def task_definitions_create(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response:
try:
payload: Any = await request.json()
if not isinstance(payload, dict):
return web.json_response(
data={"error": "Invalid request body; expected JSON object."},
status=web.HTTPBadRequest.status_code,
)
if "definition" in payload:
if not isinstance(payload["definition"], dict):
return web.json_response(
data={"error": "definition must be a JSON object when provided."},
status=web.HTTPBadRequest.status_code,
)
payload = payload["definition"]
record: TaskDefinitionRecord = task_definitions.create(payload)
except (ValueError, TypeError) as e:
return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code)
except Exception as e:
LOG.exception(e)
return web.json_response(
data={"error": "Failed to create task definition."},
status=web.HTTPInternalServerError.status_code,
)
return web.json_response(
data=record.serialize(include_definition=True),
status=web.HTTPCreated.status_code,
dumps=encoder.encode,
)
@route("PUT", "api/task_definitions/{identifier}", "task_definitions_update")
async def task_definitions_update(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response:
identifier: str = request.match_info.get("identifier", "").strip()
if not identifier:
return web.json_response(
data={"error": "Missing task definition identifier."},
status=web.HTTPBadRequest.status_code,
)
try:
payload: Any = await request.json()
if not isinstance(payload, dict):
return web.json_response(
data={"error": "Invalid request body; expected JSON object."},
status=web.HTTPBadRequest.status_code,
)
if "definition" in payload:
if not isinstance(payload["definition"], dict):
return web.json_response(
data={"error": "definition must be a JSON object when provided."},
status=web.HTTPBadRequest.status_code,
)
payload = payload["definition"]
record: TaskDefinitionRecord = task_definitions.update(identifier, payload)
except (ValueError, TypeError) as e:
return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code)
except Exception as e:
LOG.exception(e)
return web.json_response(
data={"error": "Failed to update task definition."},
status=web.HTTPInternalServerError.status_code,
)
return web.json_response(
data=record.serialize(include_definition=True),
status=web.HTTPOk.status_code,
dumps=encoder.encode,
)
@route("DELETE", "api/task_definitions/{identifier}", "task_definitions_delete")
async def task_definitions_delete(request: Request, task_definitions: TaskDefinitions) -> Response:
identifier: str = request.match_info.get("identifier", "").strip()
if not identifier:
return web.json_response(
data={"error": "Missing task definition identifier."},
status=web.HTTPBadRequest.status_code,
)
try:
task_definitions.delete(identifier)
except ValueError as e:
return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code)
except Exception as e:
LOG.exception(e)
return web.json_response(
data={"error": "Failed to delete task definition."}, status=web.HTTPInternalServerError.status_code
)
return web.json_response(data={"status": "deleted"}, status=web.HTTPOk.status_code)
import app.features.tasks.definitions.router # noqa: F401

294
app/schema/task_definition.json
View file

@ -5,11 +5,15 @@
"type": "object",
"required": [
"name",
"match",
"parse"
"match_url",
"definition"
],
"additionalProperties": false,
"properties": {
"id": {
"type": "integer",
"description": "Auto-generated unique identifier (read-only)."
},
"name": {
"type": "string",
"minLength": 1,
@ -21,49 +25,39 @@
"description": "Optional ordering priority. Lower numbers are listed first.",
"default": 0
},
"match": {
"enabled": {
"type": "boolean",
"description": "Whether this definition is active. Disabled definitions are not matched.",
"default": true
},
"match_url": {
"type": "array",
"minItems": 1,
"items": {
"oneOf": [
{
"type": "string",
"minLength": 1,
"description": "Glob pattern matched against task URLs."
},
{
"type": "object",
"additionalProperties": false,
"properties": {
"regex": {
"type": "string",
"minLength": 1,
"description": "Regular expression applied to task URLs."
},
"glob": {
"type": "string",
"minLength": 1,
"description": "Glob pattern applied to task URLs."
}
},
"anyOf": [
{
"required": [
"regex"
]
},
{
"required": [
"glob"
]
}
]
}
]
"type": "string",
"minLength": 1,
"description": "URL pattern. Use glob patterns (e.g. 'https://example.com/*') or regex (e.g. '/https://example\\.com/post/[0-9]+/')."
},
"description": "Patterns that determine which tasks use this definition."
"description": "Patterns that determine which tasks use this definition. Regex patterns are detected by /pattern/ format."
},
"engine": {
"created_at": {
"type": "string",
"format": "date-time",
"description": "Timestamp when definition was created (read-only)."
},
"updated_at": {
"type": "string",
"format": "date-time",
"description": "Timestamp when definition was last updated (read-only)."
},
"definition": {
"type": "object",
"required": [
"parse"
],
"additionalProperties": false,
"properties": {
"engine": {
"type": "object",
"additionalProperties": false,
"properties": {
@ -123,9 +117,9 @@
}
}
},
"description": "Optional engine configuration (defaults to HTTPX)."
},
"request": {
"description": "Optional engine configuration (defaults to HTTPX)."
},
"request": {
"type": "object",
"additionalProperties": false,
"properties": {
@ -165,7 +159,7 @@
}
]
},
"json": {
"json_data": {
"description": "JSON payload for POST requests.",
"type": [
"object",
@ -182,81 +176,93 @@
"description": "Timeout in seconds for the request."
}
},
"description": "Optional HTTP request overrides."
},
"response": {
"description": "Optional HTTP request overrides."
},
"response": {
"type": "object",
"additionalProperties": false,
"properties": {
"type": {
"type": "string",
"enum": [
"html",
"json"
],
"default": "html",
"description": "Body format returned by the target URL."
}
}
},
"parse": {
"type": "object",
"minProperties": 1,
"description": "Field extraction rules and optional container definition.",
"additionalProperties": false,
"properties": {
"items": {
"$ref": "#/definitions/container"
}
},
"patternProperties": {
"^(?!items$).+$": {
"$ref": "#/definitions/extractionRule"
}
},
"allOf": [
{
"if": {
"not": {
"required": [
"items"
]
"type": {
"type": "string",
"enum": [
"html",
"json"
],
"default": "html",
"description": "Body format returned by the target URL."
}
}
},
"parse": {
"type": "object",
"minProperties": 1,
"description": "Field extraction rules and optional container definition.",
"additionalProperties": false,
"properties": {
"items": {
"$ref": "#/definitions/container"
}
},
"then": {
"required": [
"link"
]
}
"patternProperties": {
"^(?!items$).+$": {
"$ref": "#/definitions/extractionRule"
}
},
"allOf": [
{
"if": {
"not": {
"required": [
"items"
]
}
},
"then": {
"required": [
"link"
]
}
}
]
}
]
}
}
},
"allOf": [
{
"if": {
"properties": {
"engine": {
"definition": {
"properties": {
"type": { "const": "selenium" }
"engine": {
"properties": {
"type": { "const": "selenium" }
},
"required": ["type"]
}
},
"required": ["type"]
"required": ["engine"]
}
},
"required": ["engine"]
"required": ["definition"]
},
"then": {
"properties": {
"engine": {
"definition": {
"properties": {
"options": {
"required": ["url"]
"engine": {
"properties": {
"options": {
"required": ["url"]
}
},
"required": ["options"]
}
},
"required": ["options"]
"required": ["engine"]
}
},
"required": ["engine"]
"required": ["definition"]
}
}
],
@ -421,66 +427,68 @@
"examples": [
{
"name": "example",
"match": [
"match_url": [
"https://example.com/articles/*",
{
"regex": "https://example.com/post/[0-9]+"
}
"/https://example\\.com/post/[0-9]+/"
],
"engine": {
"type": "httpx"
},
"request": {
"method": "GET",
"headers": {
"User-Agent": "MyCustomAgent/1.0"
}
},
"parse": {
"link": {
"type": "css",
"expression": ".article a.link",
"attribute": "href"
"definition": {
"engine": {
"type": "httpx"
},
"title": {
"type": "css",
"expression": ".article .title",
"attribute": "text"
"request": {
"method": "GET",
"headers": {
"User-Agent": "MyCustomAgent/1.0"
}
},
"id": {
"type": "regex",
"expression": "id=(?P<id>[0-9]+)",
"post_filter": {
"filter": "(?P<id>[0-9]+)",
"value": "id"
"parse": {
"link": {
"type": "css",
"expression": ".article a.link",
"attribute": "href"
},
"title": {
"type": "css",
"expression": ".article .title",
"attribute": "text"
},
"id": {
"type": "regex",
"expression": "id=(?P<id>[0-9]+)",
"post_filter": {
"filter": "(?P<id>[0-9]+)",
"value": "id"
}
}
}
}
},
{
"name": "container-example",
"match": [
"match_url": [
"https://example.com/list"
],
"parse": {
"items": {
"type": "css",
"selector": ".cards .card",
"fields": {
"link": {
"type": "css",
"expression": ".card-header a",
"attribute": "href"
},
"title": {
"type": "css",
"expression": ".card-header a",
"attribute": "text"
},
"poet": {
"type": "css",
"expression": "footer .card-footer-item:first-child a",
"attribute": "text"
"definition": {
"parse": {
"items": {
"type": "css",
"selector": ".cards .card",
"fields": {
"link": {
"type": "css",
"expression": ".card-header a",
"attribute": "href"
},
"title": {
"type": "css",
"expression": ".card-header a",
"attribute": "text"
},
"poet": {
"type": "css",
"expression": "footer .card-footer-item:first-child a",
"attribute": "text"
}
}
}
}

362
app/tests/test_generic_task_handler.py
View file

@ -1,362 +0,0 @@
import json
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from app.library.task_handlers.generic import (
ContainerDefinition,
EngineConfig,
ExtractionRule,
GenericTaskHandler,
RequestConfig,
ResponseConfig,
TaskDefinition,
load_task_definitions,
)
from app.library.Tasks import Task, TaskFailure, TaskResult
@pytest.fixture(autouse=True)
def reset_generic_handler(monkeypatch):
monkeypatch.setattr(GenericTaskHandler, "_definitions", [])
monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {})
def test_load_task_definitions_parses_valid_file(tmp_path: Path):
tasks_dir = tmp_path / "tasks"
tasks_dir.mkdir()
definition_content = {
"name": "example",
"match": ["https://example.com/articles/*"],
"parse": {
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
},
}
(tasks_dir / "01-example.json").write_text(json.dumps(definition_content), encoding="utf-8")
config = SimpleNamespace(config_path=str(tmp_path))
definitions = load_task_definitions(config=config)
assert len(definitions) == 1
definition = definitions[0]
assert definition.name == "example"
assert definition.matchers[0].matches("https://example.com/articles/123")
assert definition.parsers["link"].expression == ".article a.link::attr(href)"
def test_load_task_definitions_handles_container(tmp_path: Path):
tasks_dir = tmp_path / "tasks"
tasks_dir.mkdir()
definition_content = {
"name": "container",
"match": ["https://example.com/cards"],
"parse": {
"items": {
"selector": ".cards .card",
"fields": {
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
},
}
},
}
(tasks_dir / "02-container.json").write_text(json.dumps(definition_content), encoding="utf-8")
config = SimpleNamespace(config_path=str(tmp_path))
definitions = load_task_definitions(config=config)
assert len(definitions) == 1
definition = definitions[0]
assert definition.container is not None
assert definition.container.selector == ".cards .card"
assert "link" in definition.container.fields
assert definition.parsers == {}
def test_load_task_definitions_handles_json(tmp_path: Path):
tasks_dir = tmp_path / "tasks"
tasks_dir.mkdir()
definition_content = {
"name": "json-def",
"match": ["https://example.com/api"],
"response": {"type": "json"},
"parse": {
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
},
}
(tasks_dir / "03-json.json").write_text(json.dumps(definition_content), encoding="utf-8")
config = SimpleNamespace(config_path=str(tmp_path))
definitions = load_task_definitions(config=config)
assert len(definitions) == 1
definition = definitions[0]
assert definition.response.format == "json"
assert definition.container is not None
assert definition.container.selector_type == "jsonpath"
assert definition.container.fields["link"].type == "jsonpath"
def test_parse_items_extracts_values():
definition = TaskDefinition(
name="example",
source=Path("example.json"),
matchers=[],
engine=EngineConfig(),
request=RequestConfig(),
parsers={
"link": ExtractionRule(type="css", expression=".article a.link::attr(href)", attribute=None),
"title": ExtractionRule(type="css", expression=".article .title", attribute="text"),
"id": ExtractionRule(type="css", expression=".article", attribute="data-id"),
},
)
html = """
<div class="article" data-id="101">
<a class="link" href="/article-101">First</a>
<span class="title">First Title</span>
</div>
<div class="article" data-id="102">
<a class="link" href="https://example.com/article-102">Second</a>
<span class="title">Second Title</span>
</div>
"""
items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/")
assert len(items) == 2
assert items[0]["link"] == "https://example.com/article-101"
assert items[0]["title"] == "First Title"
assert items[0]["id"] == "101"
assert items[1]["link"] == "https://example.com/article-102"
def test_parse_items_handles_nested_card_layout():
definition = TaskDefinition(
name="nested",
source=Path("nested.json"),
matchers=[],
engine=EngineConfig(),
request=RequestConfig(),
parsers={},
container=ContainerDefinition(
selector_type="css",
selector=".columns .card",
fields={
"link": ExtractionRule(
type="css",
expression=".card-header a[href]",
attribute="href",
),
"title": ExtractionRule(
type="css",
expression=".card-header a[href]",
attribute="text",
),
"poet": ExtractionRule(
type="css",
expression="footer .card-footer-item:first-child a",
attribute="text",
),
"category": ExtractionRule(
type="css",
expression="footer .card-footer-item:nth-child(2) a",
attribute="text",
),
},
),
)
html = """
<div class="columns is-multiline">
<div class="column is-6">
<div class="card">
<div class="card-header">
<p class="card-header-title is-4 has-text-centered is-block text-truncate">
<a href="/poems/view/111" title="First Poem">First Poem</a>
</p>
</div>
<footer class="card-footer has-text-centered">
<p class="card-footer-item text-truncate">
<span class="text-truncate"> By <a href="/poet/alpha">Poet Alpha</a></span>
</p>
<p class="card-footer-item text-truncate">
<span class="text-truncate"> In <a href="/category/one">Category One</a></span>
</p>
</footer>
</div>
</div>
<div class="column is-6">
<div class="card">
<div class="card-header">
<p class="card-header-title is-4 has-text-centered is-block text-truncate">
<a href="/poems/view/222" title="Second Poem">Second Poem</a>
</p>
</div>
<footer class="card-footer has-text-centered">
<p class="card-footer-item text-truncate">
<span class="text-truncate"> By <a href="/poet/beta">Poet Beta</a></span>
</p>
</footer>
</div>
</div>
</div>
"""
items = GenericTaskHandler._parse_items(definition, html, "https://example.com")
assert len(items) == 2
assert items[0]["link"] == "https://example.com/poems/view/111"
assert items[0]["title"] == "First Poem"
assert items[0]["poet"] == "Poet Alpha"
assert items[0]["category"] == "Category One"
assert items[1]["link"] == "https://example.com/poems/view/222"
assert items[1]["title"] == "Second Poem"
assert items[1]["poet"] == "Poet Beta"
assert "category" not in items[1]
def test_parse_items_handles_json_container():
definition = TaskDefinition(
name="json",
source=Path("json.json"),
matchers=[],
engine=EngineConfig(),
request=RequestConfig(),
parsers={},
container=ContainerDefinition(
selector_type="jsonpath",
selector="entries",
fields={
"link": ExtractionRule(type="jsonpath", expression="url"),
"title": ExtractionRule(type="jsonpath", expression="title"),
"id": ExtractionRule(type="jsonpath", expression="id"),
},
),
response=ResponseConfig(format="json"),
)
payload = {
"entries": [
{"url": "/video/1", "title": "First", "id": 1},
{"url": "https://example.com/video/2", "title": "Second", "id": 2},
{"title": "Missing Link", "id": 3},
]
}
items = GenericTaskHandler._parse_items(
definition=definition,
html="",
base_url="https://example.com",
json_data=payload,
)
assert len(items) == 2
assert items[0]["link"] == "https://example.com/video/1"
assert items[0]["title"] == "First"
assert items[0]["id"] == "1"
assert items[1]["link"] == "https://example.com/video/2"
assert items[1]["title"] == "Second"
assert items[1]["id"] == "2"
@pytest.mark.asyncio
async def test_generic_task_handler_inspect(monkeypatch):
definition = TaskDefinition(
name="json-inspect",
source=Path("json-inspect.json"),
matchers=[],
engine=EngineConfig(),
request=RequestConfig(),
parsers={},
container=ContainerDefinition(
selector_type="jsonpath",
selector="items",
fields={
"link": ExtractionRule(type="jsonpath", expression="url"),
"title": ExtractionRule(type="jsonpath", expression="title"),
},
),
response=ResponseConfig(format="json"),
)
monkeypatch.setattr(
GenericTaskHandler,
"_find_definition",
classmethod(lambda cls, url: definition), # noqa: ARG005
)
async def fake_fetch_content(url, definition, ytdlp_opts): # noqa: ARG001
return "", {"items": [{"url": "/video/1", "title": "First"}]}
monkeypatch.setattr(GenericTaskHandler, "_fetch_content", staticmethod(fake_fetch_content))
# Mock fetch_info to return valid info with required fields for archive ID generation
async def fake_fetch_info(config, url, **kwargs): # noqa: ARG001
return {"id": "test_video_1", "extractor_key": "Example"}
with patch("app.library.task_handlers.generic.fetch_info", side_effect=fake_fetch_info):
task = Task(id="inspect", name="Inspect", url="https://example.com/api")
result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task)
assert isinstance(result, TaskResult)
assert len(result.items) == 1
item = result.items[0]
assert item.url == "https://example.com/video/1"
assert item.title == "First"
def test_parse_items_handles_json_top_level_list():
definition = TaskDefinition(
name="json-list",
source=Path("json-list.json"),
matchers=[],
engine=EngineConfig(),
request=RequestConfig(),
parsers={},
container=ContainerDefinition(
selector_type="jsonpath",
selector="[]",
fields={
"link": ExtractionRule(type="jsonpath", expression="url"),
"title": ExtractionRule(type="jsonpath", expression="title"),
},
),
response=ResponseConfig(format="json"),
)
payload = [
{"url": "/video/1", "title": "First"},
{"url": "/video/2", "title": "Second"},
]
items = GenericTaskHandler._parse_items(
definition=definition,
html="",
base_url="https://example.com",
json_data=payload,
)
assert len(items) == 2
assert items[0]["link"] == "https://example.com/video/1"
assert items[0]["title"] == "First"
assert items[1]["link"] == "https://example.com/video/2"
assert items[1]["title"] == "Second"

16
app/tests/test_services.py
View file

@ -3,7 +3,7 @@ from unittest.mock import MagicMock, patch
import pytest
from app.library.Services import Services
from app.library.Services import ServiceEntry, Services
class TestServices:
@ -92,9 +92,9 @@ class TestServices:
services.add("test", "value")
all_services = services.get_all()
all_services["injected"] = "malicious"
all_services.append(ServiceEntry(name="injected", declared_type=str, instance="malicious"))
assert "injected" not in services.get_all(), "Modifying returned dict should not affect internal state"
assert services.get("injected") is None, "Modifying returned dict should not affect internal state"
def test_handle_sync_with_matching_args(self):
"""Test synchronous handler with matching arguments."""
@ -138,8 +138,9 @@ class TestServices:
# Should still log error about missing arguments
mock_logger.error.assert_called_once()
error_call = mock_logger.error.call_args[0][0]
assert "Missing arguments" in error_call
assert "missing_service_param" in error_call
assert "Missing arguments for handler" in error_call, (
f"Expected 'Missing arguments for handler' in log, got: {error_call}"
)
def test_handle_sync_no_args_handler(self):
"""Test synchronous handler that takes no arguments."""
@ -197,8 +198,9 @@ class TestServices:
# Should still log error about missing arguments
mock_logger.error.assert_called_once()
error_call = mock_logger.error.call_args[0][0]
assert "Missing arguments" in error_call
assert "missing_service_param" in error_call
assert "Missing arguments for handler" in error_call, (
f"Expected 'Missing arguments for handler' in log, got: {error_call}"
)
@pytest.mark.asyncio
async def test_handle_async_no_args_handler(self):

315
app/tests/test_task_definitions.py
View file

@ -1,315 +0,0 @@
import json
import uuid
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from aiohttp import web
from aiohttp.web import Request
from jsonschema import Draft7Validator
from app.library.encoder import Encoder
from app.library.TaskDefinitions import TaskDefinitionRecord, TaskDefinitions
from app.routes.api import task_definitions as api
def _load_validator() -> Draft7Validator:
schema_path = Path(__file__).resolve().parent.parent / "schema" / "task_definition.json"
schema = json.loads(schema_path.read_text(encoding="utf-8"))
return Draft7Validator(schema)
def _sample_definition(name: str = "example", *, priority: int = 0) -> dict[str, Any]:
return {
"name": name,
"match": ["https://example.com/*"],
"priority": priority,
"parse": {
"items": {
"type": "css",
"selector": ".card",
"fields": {
"link": {"type": "css", "expression": "a", "attribute": "href"},
"title": {"type": "css", "expression": "a", "attribute": "text"},
},
}
},
}
class TestTaskDefinitionsManager:
def setup_method(self) -> None:
TaskDefinitions._reset_singleton()
def teardown_method(self) -> None:
TaskDefinitions._reset_singleton()
def test_load_populates_records(self, tmp_path: Path) -> None:
validator = _load_validator()
config = Mock(config_path=str(tmp_path), app_path=str(tmp_path))
first_identifier = "b5c6ad5f-4745-4c05-88c8-dde1deae3b51"
second_identifier = "ae38a6b0-2c22-4763-ba60-801ae8ce1218"
first = tmp_path / f"{first_identifier}.json"
second = tmp_path / f"{second_identifier}.json"
first.write_text(json.dumps(_sample_definition("First", priority=5)), encoding="utf-8")
second.write_text(json.dumps(_sample_definition("Second", priority=1)), encoding="utf-8")
(tmp_path / "not-a-uuid.json").write_text(json.dumps(_sample_definition("Ignored")), encoding="utf-8")
(tmp_path / "0f9184de-5d3c-2111-8c21-6d3f0be1bd3d.json").write_text(
json.dumps(_sample_definition("Ignored2")),
encoding="utf-8",
)
manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator)
records = manager.list()
assert len(records) == 2
assert [record.name for record in records] == ["Second", "First"]
assert [record.priority for record in records] == [1, 5]
def test_create_writes_file_and_refreshes(self, tmp_path: Path) -> None:
validator = _load_validator()
config = Mock(config_path=str(tmp_path), app_path=str(tmp_path))
manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator)
with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh:
definition = _sample_definition("My Definition")
record = manager.create(definition)
refresh.assert_called_once_with(force=True)
identifier_uuid = uuid.UUID(record.identifier)
assert 4 == identifier_uuid.version
expected_filename = f"{record.identifier}.json"
saved_path = tmp_path / expected_filename
assert saved_path.exists()
saved_content = json.loads(saved_path.read_text(encoding="utf-8"))
assert saved_content["name"] == "My Definition"
assert saved_content["priority"] == 0
def test_update_missing_definition_raises(self, tmp_path: Path) -> None:
validator = _load_validator()
config = Mock(config_path=str(tmp_path), app_path=str(tmp_path))
manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator)
with pytest.raises(ValueError, match="does not exist"):
manager.update("missing", _sample_definition("Updated"))
def test_update_overwrites_file(self, tmp_path: Path) -> None:
validator = _load_validator()
config = Mock(config_path=str(tmp_path), app_path=str(tmp_path))
identifier = "c59ec7cf-6291-4f0f-86f8-d8cb12c325a4"
initial = _sample_definition("Original", priority=4)
path = tmp_path / f"{identifier}.json"
path.write_text(json.dumps(initial), encoding="utf-8")
manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator)
with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh:
updated_record = manager.update(identifier, _sample_definition("Updated", priority=2))
refresh.assert_called_once_with(force=True)
assert updated_record.name == "Updated"
assert updated_record.priority == 2
saved = json.loads(path.read_text(encoding="utf-8"))
assert saved["name"] == "Updated"
assert saved["priority"] == 2
def test_delete_removes_file(self, tmp_path: Path) -> None:
validator = _load_validator()
config = Mock(config_path=str(tmp_path), app_path=str(tmp_path))
identifier = "f0b71f47-6b65-4b6d-89fd-6b87ce47d3bc"
definition_path = tmp_path / f"{identifier}.json"
definition_path.write_text(json.dumps(_sample_definition("Delete")), encoding="utf-8")
manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator)
with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh:
manager.delete(identifier)
refresh.assert_called_once_with(force=True)
assert not definition_path.exists()
assert manager.get(identifier) is None
@pytest.mark.asyncio
class TestTaskDefinitionRoutes:
def setup_method(self) -> None:
TaskDefinitions._reset_singleton()
def teardown_method(self) -> None:
TaskDefinitions._reset_singleton()
async def test_list_definitions(self) -> None:
request = MagicMock(spec=Request)
request.query = {}
identifier = "9af7018f-8659-4d2a-a42b-b5d2c5f0a6e2"
record = TaskDefinitionRecord(
identifier=identifier,
filename=f"{identifier}.json",
name="Sample",
priority=0,
path=Path("/tmp/sample.json"),
data=_sample_definition("Sample"),
updated_at=123.0,
)
task_definitions = MagicMock(spec=TaskDefinitions)
task_definitions.list.return_value = [record]
response = await api.task_definitions_list(request, Encoder(), task_definitions)
payload = json.loads(response.text)
assert response.status == web.HTTPOk.status_code
assert payload == [record.serialize()]
assert "filename" not in payload[0]
async def test_list_definitions_includes_definition(self) -> None:
request = MagicMock(spec=Request)
request.query = {"include": "definition"}
identifier = "f5b5e88d-5c6b-4a27-8453-1b6a4fb8a8d1"
record = TaskDefinitionRecord(
identifier=identifier,
filename=f"{identifier}.json",
name="Sample",
priority=0,
path=Path("/tmp/sample.json"),
data=_sample_definition("Sample"),
updated_at=123.0,
)
task_definitions = MagicMock(spec=TaskDefinitions)
task_definitions.list.return_value = [record]
response = await api.task_definitions_list(request, Encoder(), task_definitions)
payload = json.loads(response.text)
assert payload[0]["definition"]["name"] == "Sample"
async def test_get_definition_not_found(self) -> None:
request = MagicMock(spec=Request)
request.match_info = {"identifier": "unknown"}
task_definitions = MagicMock(spec=TaskDefinitions)
task_definitions.get.return_value = None
response = await api.task_definitions_get(request, Encoder(), task_definitions)
payload = json.loads(response.text)
assert response.status == web.HTTPNotFound.status_code
assert "error" in payload
async def test_create_definition_success(self) -> None:
payload_definition = _sample_definition("New", priority=1)
payload = {"definition": payload_definition}
request = MagicMock(spec=Request)
request.json = AsyncMock(return_value=payload)
identifier = "4f08b8af-b87a-4d6e-9289-39e5172898aa"
record = TaskDefinitionRecord(
identifier=identifier,
filename=f"{identifier}.json",
name="New",
priority=1,
path=Path("/tmp/new.json"),
data=payload["definition"],
updated_at=123.0,
)
task_definitions = MagicMock(spec=TaskDefinitions)
task_definitions.create.return_value = record
response = await api.task_definitions_create(request, Encoder(), task_definitions)
body = json.loads(response.text)
assert response.status == web.HTTPCreated.status_code
assert body["id"] == identifier
assert body["priority"] == 1
assert "filename" not in body
task_definitions.create.assert_called_once_with(payload_definition)
async def test_create_definition_invalid_payload(self) -> None:
request = MagicMock(spec=Request)
request.json = AsyncMock(return_value=[]) # type: ignore[arg-type]
task_definitions = MagicMock(spec=TaskDefinitions)
response = await api.task_definitions_create(request, Encoder(), task_definitions)
body = json.loads(response.text)
assert response.status == web.HTTPBadRequest.status_code
assert "error" in body
async def test_update_definition_success(self) -> None:
request = MagicMock(spec=Request)
identifier = "6d8d5719-95ae-4478-bb05-986f5b72b6c1"
request.match_info = {"identifier": identifier}
definition = _sample_definition("Updated", priority=4)
request.json = AsyncMock(return_value={"definition": definition})
record = TaskDefinitionRecord(
identifier=identifier,
filename=f"{identifier}.json",
name="Updated",
priority=4,
path=Path("/tmp/existing.json"),
data=definition,
updated_at=456.0,
)
task_definitions = MagicMock(spec=TaskDefinitions)
task_definitions.update.return_value = record
response = await api.task_definitions_update(request, Encoder(), task_definitions)
body = json.loads(response.text)
assert response.status == web.HTTPOk.status_code
assert body["name"] == "Updated"
assert body["priority"] == 4
task_definitions.update.assert_called_once_with(identifier, definition)
async def test_update_definition_missing_identifier(self) -> None:
request = MagicMock(spec=Request)
request.match_info = {"identifier": ""}
request.json = AsyncMock(return_value={})
task_definitions = MagicMock(spec=TaskDefinitions)
response = await api.task_definitions_update(request, Encoder(), task_definitions)
body = json.loads(response.text)
assert response.status == web.HTTPBadRequest.status_code
assert "error" in body
async def test_delete_definition_success(self) -> None:
request = MagicMock(spec=Request)
identifier = "c9f4ac6c-a4ab-4d1a-8d25-764dc0c8a3f0"
request.match_info = {"identifier": identifier}
task_definitions = MagicMock(spec=TaskDefinitions)
response = await api.task_definitions_delete(request, task_definitions)
body = json.loads(response.text)
assert response.status == web.HTTPOk.status_code
assert body["status"] == "deleted"
task_definitions.delete.assert_called_once_with(identifier)
async def test_delete_definition_missing_identifier(self) -> None:
request = MagicMock(spec=Request)
request.match_info = {"identifier": ""}
task_definitions = MagicMock(spec=TaskDefinitions)
response = await api.task_definitions_delete(request, task_definitions)
body = json.loads(response.text)
assert response.status == web.HTTPBadRequest.status_code
assert "error" in body

13
app/tests/test_tasks.py
View file

@ -1053,7 +1053,8 @@ class TestHandleTaskInspect:
# Mock Services to simulate successful can_handle
mock_services_instance = Mock()
mock_services_instance.handle_sync = Mock(return_value=True)
mock_services_instance.handle_async = AsyncMock() # Should NOT be called with static_only=True
# handle_async will be called once for can_handle (since it's now async)
mock_services_instance.handle_async = AsyncMock(return_value=True)
mock_services.get_instance.return_value = mock_services_instance
# Create a mock handler
@ -1077,8 +1078,8 @@ class TestHandleTaskInspect:
assert result.metadata["matched"] is True
assert result.metadata["handler"] == "TestHandler"
# Verify handle_async (extract) was NOT called
mock_services_instance.handle_async.assert_not_called()
# Verify handle_async was called once for can_handle (now async)
assert mock_services_instance.handle_async.call_count == 1, "Should call handle_async once for can_handle"
@pytest.mark.asyncio
@patch("app.library.Tasks.Config")
@ -1132,8 +1133,10 @@ class TestHandleTaskInspect:
assert result.metadata["handler"] == "TestHandler"
assert result.metadata["supported"] is True
# Verify handle_async (extract) WAS called
mock_services_instance.handle_async.assert_called_once()
# Verify handle_async was called twice: once for can_handle, once for extract
assert mock_services_instance.handle_async.call_count == 2, (
"Should call handle_async twice: can_handle + extract"
)
@pytest.mark.asyncio
@patch("app.library.Tasks.Config")

2
app/tests/test_twitch_handler.py
View file

@ -1,6 +1,6 @@
import pytest
from app.library.task_handlers.twitch import TwitchHandler
from app.features.tasks.definitions.handlers.twitch import TwitchHandler
from app.library.Tasks import Task, TaskResult

2
app/tests/test_youtube_handler.py
View file

@ -1,6 +1,6 @@
import pytest
from app.library.task_handlers.youtube import YoutubeHandler
from app.features.tasks.definitions.handlers.youtube import YoutubeHandler
from app.library.Tasks import Task, TaskResult

2
pyproject.toml
View file

@ -186,7 +186,7 @@ pythonpath = ["."]
testpaths = ["app/tests", "app/features"]
addopts = "-v --tb=short"
filterwarnings = [
"ignore:Parsing dates involving a day of month without a year:DeprecationWarning",
"ignore:Parsing dates involving a day of month without a year:DeprecationWarning"
]
[dependency-groups]

150
ui/app/components/TaskDefinitionEditor.vue
View file

@ -32,7 +32,7 @@
<div class="card-content">
<div class="columns is-multiline" v-if="showImport">
<div class="column is-12" v-if="availableDefinitions.length">
<div class="column is-6-tablet is-12-mobile" v-if="availableDefinitions.length">
<div class="field">
<label class="label is-inline">
<span class="icon"><i class="fa-solid fa-diagram-project" /></span>
@ -51,13 +51,14 @@
<p class="help is-bold">
<span class="icon-text">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>Loads an existing definition into the editor. Changes are not saved until you submit.</span>
<span>Pre-fill from an existing task definition.</span>
</span>
</p>
</div>
</div>
<div class="column is-12">
<div class="column"
:class="{ 'is-6-tablet is-12-mobile': availableDefinitions.length, 'is-12': !availableDefinitions.length }">
<div class="field">
<label class="label is-inline">
<span class="icon"><i class="fa-solid fa-file-import" /></span>
@ -78,7 +79,7 @@
<p class="help is-bold">
<span class="icon-text">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>Pastes a shared task definition string. Importing replaces the current editor contents.</span>
<span>Paste shared task definition string here to import it.</span>
</span>
</p>
</div>
@ -125,6 +126,25 @@
</div>
</div>
<div class="column is-3">
<div class="field">
<label class="label is-inline">
<span class="icon"><i class="fa-solid fa-toggle-on" /></span>
Status
</label>
<div class="control">
<label class="checkbox">
<input type="checkbox" v-model="guiState.enabled" :disabled="isBusy">
<span class="ml-2">Enabled</span>
</label>
</div>
<p class="help is-bold">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>Disabled definitions won't match tasks.</span>
</p>
</div>
</div>
<div class="column is-12">
<div class="field">
<label class="label is-inline">
@ -137,7 +157,7 @@
</div>
<p class="help is-bold">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>One glob per line. Regex or object based rules are only supported in advanced mode.</span>
<span>One glob/regex url per line</span>
</p>
</div>
</div>
@ -377,6 +397,7 @@ type GuiField = {
type GuiState = {
name: string
priority: number
enabled: boolean
matchText: string
engineType: 'httpx' | 'selenium'
engineUrl: string
@ -399,7 +420,7 @@ const props = defineProps<{
const emit = defineEmits<{
(e: 'submit', payload: TaskDefinitionDocument): void
(e: 'cancel'): void
(e: 'import-existing', id: string): void
(e: 'import-existing', id: number): void
}>()
const jsonText = ref<string>('')
@ -409,13 +430,14 @@ const guiSupported = ref<boolean>(true)
const mode = ref<EditorMode>('gui')
const showImport = ref<boolean>(false)
const importString = ref<string>('')
const selectedExisting = ref<string>('')
const selectedExisting = ref<number | ''>('')
const availableDefinitions = computed<readonly TaskDefinitionSummary[]>(() => props.availableDefinitions ?? [])
const guiState = reactive<GuiState>({
name: '',
priority: 0,
enabled: true,
matchText: '',
engineType: 'httpx',
engineUrl: '',
@ -431,12 +453,13 @@ const submitting = computed<boolean>(() => props.submitting ?? false)
const isBusy = computed<boolean>(() => loading.value || submitting.value)
const headerTitle = computed<string>(() => props.title)
const guiLimitations = 'Only simple match globs, a single container selector and per-field extractors are exposed. ' +
const guiLimitations = 'Only a single container selector and per-field extractors are exposed. ' +
'More advanced constructs require raw view mode.'
const resetGuiState = (state: GuiState): void => {
guiState.name = state.name
guiState.priority = state.priority
guiState.enabled = state.enabled
guiState.matchText = state.matchText
guiState.engineType = state.engineType
guiState.engineUrl = state.engineUrl
@ -467,12 +490,17 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => {
}
const entry = document
const match = entry.match
const match = entry.match_url
if (!Array.isArray(match) || match.some(item => 'string' !== typeof item)) {
return null
}
const parse = entry.parse
const definition = entry.definition
if (!definition || Array.isArray(definition) || 'object' !== typeof definition) {
return null
}
const parse = definition.parse
if (!parse || Array.isArray(parse) || 'object' !== typeof parse) {
return null
}
@ -499,7 +527,7 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => {
if ('string' !== typeof rule.type || 'string' !== typeof rule.expression) {
return null
}
if (Object.keys(rule).some(prop => !['type', 'expression', 'attribute'].includes(prop))) {
if (Object.keys(rule).some(prop => !['type', 'expression', 'attribute', 'post_filter'].includes(prop))) {
return null
}
guiFields.push({
@ -510,7 +538,7 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => {
})
}
const engine = entry.engine as Record<string, unknown> | undefined
const engine = definition.engine as Record<string, unknown> | undefined
const engineType = (engine?.type === 'selenium') ? 'selenium' : 'httpx'
const engineUrl = 'string' === typeof engine?.options && engineType === 'selenium'
? ''
@ -520,7 +548,7 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => {
return null
}
const request = entry.request as Record<string, unknown> | undefined
const request = definition.request as Record<string, unknown> | undefined
const selectorType = String(itemRecord.type ?? 'css') as GuiState['containerType']
const selectorSource = (itemRecord.selector ?? itemRecord.expression) as string | undefined
@ -531,6 +559,7 @@ const toGui = (document: TaskDefinitionDocument): GuiState | null => {
return {
name: 'string' === typeof entry.name ? entry.name : '',
priority: Number(entry.priority ?? 0) || 0,
enabled: 'boolean' === typeof entry.enabled ? entry.enabled : true,
matchText: match.join('\n'),
engineType,
engineUrl: engineType === 'selenium' ? String(engineUrl ?? '') : '',
@ -575,10 +604,7 @@ const fromGui = (state: GuiState): TaskDefinitionDocument => {
throw new Error('Configure at least one extractor field.')
}
const doc: Record<string, unknown> = {
name: state.name.trim(),
priority: Number(state.priority) || 0,
match: matches,
const definition: Record<string, unknown> = {
parse: {
items: {
type: state.containerType,
@ -590,7 +616,7 @@ const fromGui = (state: GuiState): TaskDefinitionDocument => {
}
if ('httpx' !== state.engineType || state.engineUrl) {
doc.engine = {
definition.engine = {
type: state.engineType,
...(state.engineType === 'selenium' && state.engineUrl
? { options: { url: state.engineUrl } }
@ -606,10 +632,31 @@ const fromGui = (state: GuiState): TaskDefinitionDocument => {
request.url = state.requestUrl
}
if (Object.keys(request).length) {
doc.request = request
definition.request = request
}
return doc as unknown as TaskDefinitionDocument
return {
name: state.name.trim(),
priority: Number(state.priority) || 0,
enabled: state.enabled,
match_url: matches,
definition: definition as unknown as TaskDefinitionDocument['definition'],
}
}
const normalizeRequestConfig = (request: any): any => {
if (!request || 'object' !== typeof request) {
return request
}
if ('json' in request) {
const normalized = { ...request }
normalized.json_data = normalized.json
delete normalized.json
return normalized
}
return request
}
const parseImportedDocument = (payload: unknown): TaskDefinitionDocument => {
@ -618,31 +665,59 @@ const parseImportedDocument = (payload: unknown): TaskDefinitionDocument => {
}
const record = payload as Record<string, unknown>
const candidate = record.definition
if ('_type' in record && record._type !== undefined && record._type !== 'task_definition') {
throw new Error('Import string is not a task definition export.')
}
const version = record._version as string | undefined
if (-1 === ['1.0', '2.0'].indexOf(version ?? '')) {
throw new Error(`Unsupported or missing _version field. Expected "1.0" or "2.0", got: ${version ?? 'undefined'}`)
}
let base: TaskDefinitionDocument
if (candidate && !Array.isArray(candidate) && 'object' === typeof candidate) {
base = candidate as TaskDefinitionDocument
if ('1.0' === version) {
// v1.0 format migration
const oldDef = record.definition as Record<string, unknown>
// Normalize match_url from old v1.0 format
const oldMatch = Array.isArray(oldDef.match) ? oldDef.match : []
const normalizedMatch: string[] = []
for (const item of oldMatch) {
if ('string' === typeof item) {
normalizedMatch.push(item)
}
else if ('object' === typeof item && item !== null) {
const obj = item as Record<string, unknown>
if ('string' === typeof obj.regex) {
normalizedMatch.push(`/${obj.regex}/`)
}
else if ('string' === typeof obj.glob) {
normalizedMatch.push(obj.glob)
}
}
}
base = {
name: 'string' === typeof oldDef.name ? oldDef.name : 'string' === typeof record.name ? record.name : '',
priority: Number(oldDef.priority ?? record.priority ?? 0) || 0,
enabled: true,
match_url: normalizedMatch,
definition: {
parse: oldDef.parse as any,
engine: oldDef.engine as any,
request: normalizeRequestConfig(oldDef.request),
response: oldDef.response as any,
},
}
}
else {
base = payload as TaskDefinitionDocument
base = record as unknown as TaskDefinitionDocument
}
const clone = JSON.parse(JSON.stringify(base)) as TaskDefinitionDocument
const cloneRecord = clone
if ('name' in record && 'string' === typeof record.name) {
cloneRecord.name = record.name
}
if ('priority' in record && record.priority !== undefined) {
cloneRecord.priority = Number(record.priority) || 0
}
return clone
return JSON.parse(JSON.stringify(base)) as TaskDefinitionDocument
}
const parseDocument = (): TaskDefinitionDocument | null => {
@ -675,6 +750,7 @@ const applyDocument = (document: TaskDefinitionDocument | null): void => {
resetGuiState({
name: '',
priority: 0,
enabled: true,
matchText: '',
engineType: 'httpx',
engineUrl: '',
@ -741,7 +817,7 @@ const importExisting = (): void => {
return
}
emit('import-existing', selectedExisting.value)
emit('import-existing', Number(selectedExisting.value))
selectedExisting.value = ''
}

173
ui/app/composables/useTaskDefinitions.ts
View file

@ -1,18 +1,29 @@
import { ref, readonly } from 'vue'
import { useNotification } from '~/composables/useNotification'
import { request } from '~/utils'
import { request, parse_list_response, parse_api_response, parse_api_error } from '~/utils'
import type {
TaskDefinitionDetailed,
TaskDefinitionDocument,
TaskDefinitionErrorResponse,
TaskDefinitionSummary,
} from '~/types/task_definitions'
import type { Pagination } from '~/types/responses'
/**
* Reactive list of all task definition summaries, sorted by priority and name.
*/
const definitions = ref<Array<TaskDefinitionSummary>>([])
/**
* Pagination state for task definitions list.
*/
const pagination = ref<Pagination>({
page: 1,
per_page: 50,
total: 0,
total_pages: 0,
has_next: false,
has_prev: false,
})
/**
* Indicates if a request is in progress.
*/
@ -47,21 +58,6 @@ const sortSummaries = (items: Array<TaskDefinitionSummary>): Array<TaskDefinitio
})
}
/**
* Safely reads JSON from a Response, returns null on error.
* @param response Fetch Response object
* @returns Parsed JSON or null
*/
const readJson = async (response: Response): Promise<unknown> => {
try {
const clone = response.clone()
return await clone.json()
}
catch {
return null
}
}
/**
* Throws an error if the response is not OK, using API error message if available.
* @param response Fetch Response object
@ -72,16 +68,8 @@ const ensureSuccess = async (response: Response): Promise<void> => {
return
}
const payload = await readJson(response)
let message = `Request failed with status ${response.status}`
if (payload && typeof payload === 'object' && 'error' in payload) {
const errorPayload = payload as TaskDefinitionErrorResponse
if (typeof errorPayload.error === 'string' && errorPayload.error.length > 0) {
message = errorPayload.error
}
}
const payload = await response.clone().json().catch(() => null)
const message = await parse_api_error(payload)
throw new Error(message)
}
@ -100,48 +88,47 @@ const handleError = (error: unknown): void => {
* @param summary TaskDefinitionSummary to update/add
*/
const updateSummaries = (summary: TaskDefinitionSummary): void => {
const isNew = !definitions.value.some(item => item.id === summary.id)
definitions.value = sortSummaries([
...definitions.value.filter(item => item.id !== summary.id),
summary,
])
if (isNew) {
pagination.value.total++
}
}
/**
* Removes a summary from the definitions list by ID.
* @param id Task definition ID
*/
const removeSummary = (id: string) => definitions.value = definitions.value.filter(item => item.id !== id)
const removeSummary = (id: number) => {
const initialLength = definitions.value.length
definitions.value = definitions.value.filter(item => item.id !== id)
if (definitions.value.length < initialLength) {
pagination.value.total = Math.max(0, pagination.value.total - 1)
}
}
/**
* Loads all task definition summaries from the API.
* Updates definitions and lastError.
*/
const loadDefinitions = async (): Promise<void> => {
const loadDefinitions = async (page: number = 1, perPage: number | undefined = undefined): Promise<void> => {
isLoading.value = true
try {
const response = await request('/api/task_definitions/')
let url = `/api/tasks/definitions/?page=${page}`
if (perPage !== undefined) {
url += `&per_page=${perPage}`
}
const response = await request(url)
await ensureSuccess(response)
const payload = await response.json() as unknown
if (!Array.isArray(payload)) {
throw new Error('Unexpected response while loading task definitions.')
}
const json = await response.json()
const { items, pagination: paginationData } = await parse_list_response<TaskDefinitionSummary>(json)
const summaries: Array<TaskDefinitionSummary> = payload.map(item => {
if (!item || 'object' !== typeof item) {
throw new Error('Encountered malformed task definition entry.')
}
const entry = item as Record<string, unknown>
return {
id: String(entry.id ?? ''),
name: String(entry.name ?? ''),
priority: Number(entry.priority ?? 0),
updated_at: Number(entry.updated_at ?? 0),
}
})
definitions.value = sortSummaries(summaries)
definitions.value = sortSummaries(items)
pagination.value = paginationData
lastError.value = null
}
catch (error) {
@ -158,29 +145,13 @@ const loadDefinitions = async (): Promise<void> => {
* @param id Task definition ID
* @returns TaskDefinitionDetailed or null on error
*/
const getDefinition = async (id: string): Promise<TaskDefinitionDetailed | null> => {
const getDefinition = async (id: number): Promise<TaskDefinitionDetailed | null> => {
try {
const response = await request(`/api/task_definitions/${id}`)
const response = await request(`/api/tasks/definitions/${id}`)
await ensureSuccess(response)
const payload = await response.json() as unknown
if (!payload || 'object' !== typeof payload) {
throw new Error('Unexpected response while retrieving task definition.')
}
const entry = payload as Record<string, unknown>
if (!('definition' in entry) || 'object' !== typeof entry.definition) {
throw new Error('Task definition response is missing definition payload.')
}
const detailed: TaskDefinitionDetailed = {
id: String(entry.id ?? ''),
name: String(entry.name ?? ''),
priority: Number(entry.priority ?? 0),
updated_at: Number(entry.updated_at ?? 0),
definition: entry.definition as TaskDefinitionDocument,
}
const payload = await response.json()
const detailed = await parse_api_response<TaskDefinitionDetailed>(payload)
lastError.value = null
return detailed
}
@ -198,19 +169,22 @@ const getDefinition = async (id: string): Promise<TaskDefinitionDetailed | null>
*/
const createDefinition = async (definition: TaskDefinitionDocument): Promise<TaskDefinitionDetailed | null> => {
try {
const response = await request('/api/task_definitions/', {
const response = await request('/api/tasks/definitions/', {
method: 'POST',
body: JSON.stringify({ definition }),
body: JSON.stringify(definition),
})
await ensureSuccess(response)
const payload = await response.json() as TaskDefinitionDetailed
const payload = await parse_api_response<TaskDefinitionDetailed>(response.json())
updateSummaries({
id: payload.id,
name: payload.name,
priority: payload.priority,
match_url: payload.match_url,
enabled: payload.enabled,
created_at: payload.created_at,
updated_at: payload.updated_at,
})
@ -231,21 +205,24 @@ const createDefinition = async (definition: TaskDefinitionDocument): Promise<Tas
* @param definition Updated TaskDefinitionDocument
* @returns Updated TaskDefinitionDetailed or null on error
*/
const updateDefinition = async (id: string, definition: TaskDefinitionDocument): Promise<TaskDefinitionDetailed | null> => {
const updateDefinition = async (id: number, definition: TaskDefinitionDocument): Promise<TaskDefinitionDetailed | null> => {
try {
const response = await request(`/api/task_definitions/${id}`, {
const response = await request(`/api/tasks/definitions/${id}`, {
method: 'PUT',
body: JSON.stringify({ definition }),
body: JSON.stringify(definition),
})
await ensureSuccess(response)
const payload = await response.json() as TaskDefinitionDetailed
const payload = await parse_api_response<TaskDefinitionDetailed>(response.json())
updateSummaries({
id: payload.id,
name: payload.name,
priority: payload.priority,
match_url: payload.match_url,
enabled: payload.enabled,
created_at: payload.created_at,
updated_at: payload.updated_at,
})
@ -265,9 +242,9 @@ const updateDefinition = async (id: string, definition: TaskDefinitionDocument):
* @param id Task definition ID
* @returns true if deleted, false on error
*/
const deleteDefinition = async (id: string): Promise<boolean> => {
const deleteDefinition = async (id: number): Promise<boolean> => {
try {
const response = await request(`/api/task_definitions/${id}`, { method: 'DELETE' })
const response = await request(`/api/tasks/definitions/${id}`, { method: 'DELETE' })
await ensureSuccess(response)
removeSummary(id)
@ -282,6 +259,44 @@ const deleteDefinition = async (id: string): Promise<boolean> => {
}
}
/**
* Toggles the enabled status of a task definition.
* @param id Task definition ID
* @param enabled New enabled status
* @returns Updated TaskDefinitionDetailed or null on error
*/
const toggleEnabled = async (id: number, enabled: boolean): Promise<TaskDefinitionDetailed | null> => {
try {
const response = await request(`/api/tasks/definitions/${id}`, {
method: 'PATCH',
body: JSON.stringify({ enabled }),
})
await ensureSuccess(response)
const payload = await parse_api_response<TaskDefinitionDetailed>(response.json())
updateSummaries({
id: payload.id,
name: payload.name,
priority: payload.priority,
match_url: payload.match_url,
enabled: payload.enabled,
created_at: payload.created_at,
updated_at: payload.updated_at,
})
notify.success(`Task definition ${enabled ? 'enabled' : 'disabled'}.`)
lastError.value = null
return payload
}
catch (error) {
handleError(error)
if (throwInstead.value) throw error
return null
}
}
/**
* Clears the last error message.
*/
@ -295,6 +310,7 @@ const clearError = () => lastError.value = null
*/
export const useTaskDefinitions = () => ({
definitions: readonly(definitions),
pagination: readonly(pagination),
isLoading: readonly(isLoading),
lastError: readonly(lastError),
loadDefinitions,
@ -302,6 +318,7 @@ export const useTaskDefinitions = () => ({
createDefinition,
updateDefinition,
deleteDefinition,
toggleEnabled,
clearError,
throwInstead,
})

111
ui/app/pages/task_definitions.vue
View file

@ -75,14 +75,23 @@
</thead>
<tbody>
<tr v-for="definition in definitions" :key="definition.id">
<td class="is-vcentered is-text-overflow">
{{ definition.name || '(Unnamed definition)' }}
<td class="is-vcentered">
<div class="is-text-overflow">{{ definition.name || '(Unnamed definition)' }}</div>
<div class="is-size-7">
<span class="icon-text is-clickable" @click="toggle(definition)"
v-tooltip="'Click to ' + (definition.enabled ? 'disable' : 'enable') + ' definition'">
<span class="icon">
<i class="fa-solid fa-power-off"
:class="{ 'has-text-success': definition.enabled, 'has-text-danger': !definition.enabled }" />
</span>
<span>{{ definition.enabled ? 'Enabled' : 'Disabled' }}</span>
</span>
</div>
</td>
<td class="is-vcentered has-text-centered">{{ definition.priority }}</td>
<td class="is-vcentered has-text-centered">
<span class="has-tooltip"
:date-datetime="moment.unix(definition.updated_at).format('YYYY-M-DD H:mm Z')"
v-tooltip="moment.unix(definition.updated_at).format('YYYY-M-DD H:mm Z')"
<span class="has-tooltip" :date-datetime="moment(definition.updated_at).format('YYYY-M-DD H:mm Z')"
v-tooltip="moment(definition.updated_at).format('YYYY-M-DD H:mm Z')"
v-rtime="definition.updated_at" />
</td>
<td class="is-vcentered is-items-center">
@ -122,9 +131,19 @@
{{ definition.name || '(Unnamed definition)' }}
</div>
<div class="card-header-icon">
<button class="has-text-info" v-tooltip="'Export'" @click="exportDefinition(definition)">
<span class="icon"><i class="fa-solid fa-file-export" /></span>
</button>
<div class="field has-addons">
<div class="control" @click="toggle(definition)">
<span class="icon" :class="definition.enabled ? 'has-text-success' : 'has-text-danger'"
v-tooltip="`Definition is ${definition.enabled ? 'enabled' : 'disabled'}. Click to toggle.`">
<i class="fa-solid fa-power-off" />
</span>
</div>
<div class="control">
<button class="has-text-info" v-tooltip="'Export'" @click="exportDefinition(definition)">
<span class="icon"><i class="fa-solid fa-file-export" /></span>
</button>
</div>
</div>
</div>
</header>
<div class="card-content">
@ -139,8 +158,8 @@
<span class="icon-text">
<span class="icon"><i class="fa-solid fa-clock" /></span>
<span>Updated: <span class="has-tooltip"
:date-datetime="moment.unix(definition.updated_at).format('YYYY-M-DD H:mm Z')"
v-tooltip="moment.unix(definition.updated_at).format('YYYY-M-DD H:mm Z')"
:date-datetime="moment(definition.updated_at).format('YYYY-M-DD H:mm Z')"
v-tooltip="moment(definition.updated_at).format('YYYY-M-DD H:mm Z')"
v-rtime="definition.updated_at" />
</span>
</span>
@ -219,14 +238,17 @@ import type {
const DEFAULT_DEFINITION: TaskDefinitionDocument = {
name: 'New Definition',
priority: 0,
match: ['https://example.com/*'],
parse: {
items: {
type: 'css',
selector: 'body',
fields: {
link: { type: 'css', expression: 'a', attribute: 'href' },
title: { type: 'css', expression: 'a', attribute: 'text' },
enabled: true,
match_url: ['https://example.com/*'],
definition: {
parse: {
items: {
type: 'css',
selector: 'body',
fields: {
link: { type: 'css', expression: 'a', attribute: 'href' },
title: { type: 'css', expression: 'a', attribute: 'text' },
},
},
},
},
@ -242,6 +264,7 @@ const getDefinition = taskDefs.getDefinition
const createDefinition = taskDefs.createDefinition
const updateDefinition = taskDefs.updateDefinition
const deleteDefinition = taskDefs.deleteDefinition
const toggleEnabled = taskDefs.toggleEnabled
const definitions = computed(() => definitionsRef.value)
@ -253,7 +276,7 @@ const editorMode = ref<'create' | 'edit'>('create')
const editorLoading = ref<boolean>(false)
const editorSubmitting = ref<boolean>(false)
const workingDefinition = ref<TaskDefinitionDocument | null>(null)
const workingId = ref<string | null>(null)
const workingId = ref<number | null>(null)
const inspect = ref<boolean>(false)
const display_style = useStorage<'list' | 'grid'>('task-definitions:display', 'grid')
@ -299,39 +322,37 @@ const openEdit = async (summary: TaskDefinitionSummary): Promise<void> => {
return
}
const document = cloneDocument(detailed.definition)
const docRecord = document
if ('priority' in docRecord) {
docRecord.priority = Number(docRecord.priority)
}
else {
docRecord.priority = detailed.priority
const document: TaskDefinitionDocument = {
name: detailed.name,
priority: detailed.priority,
enabled: detailed.enabled,
match_url: [...detailed.match_url],
definition: JSON.parse(JSON.stringify(detailed.definition)),
}
workingDefinition.value = document
editorLoading.value = false
}
const importExistingDefinition = async (id: string): Promise<void> => {
const importExistingDefinition = async (id: number): Promise<void> => {
const detailed = await getDefinition(id)
if (!detailed) {
toast.error('Failed to load task definition for import.')
return
}
const document = cloneDocument(detailed.definition)
const docRecord = document
if ('priority' in docRecord) {
docRecord.priority = Number(docRecord.priority)
}
else {
docRecord.priority = detailed.priority
const document: TaskDefinitionDocument = {
name: detailed.name,
priority: detailed.priority,
enabled: detailed.enabled,
match_url: [...detailed.match_url],
definition: JSON.parse(JSON.stringify(detailed.definition)),
}
editorMode.value = 'create'
workingId.value = null
workingDefinition.value = document
if ('create' === editorMode.value) {
workingId.value = null
}
isEditorOpen.value = true
editorLoading.value = false
}
@ -374,7 +395,7 @@ const submitDefinition = async (definition: TaskDefinitionDocument): Promise<voi
const remove = async (summary: TaskDefinitionSummary): Promise<void> => {
const result = await confirmDialog({
title: 'Delete Task Definition',
message: `Are you sure you want to delete “${summary.name || summary.id}”?`,
message: `Are you sure you want to delete "${summary.name || summary.id}"?`,
confirmColor: 'is-danger',
})
@ -385,21 +406,25 @@ const remove = async (summary: TaskDefinitionSummary): Promise<void> => {
await deleteDefinition(summary.id)
}
const toggle = async (summary: TaskDefinitionSummary): Promise<void> => {
await toggleEnabled(summary.id, !summary.enabled)
}
const exportDefinition = async (summary: TaskDefinitionSummary): Promise<void> => {
const detailed = await getDefinition(summary.id)
if (!detailed) {
return
}
const payload = {
return copyText(encode({
_type: 'task_definition',
_version: '1.0',
_version: '2.0',
name: detailed.name,
priority: detailed.priority,
enabled: detailed.enabled,
match_url: detailed.match_url,
definition: detailed.definition,
}
return copyText(encode(payload))
}))
}
onMounted(async () => {

35
ui/app/types/task_definitions.ts
View file

@ -1,6 +1,5 @@
// --- Task Definition Schema Types ---
export type TaskMatchPattern = | string | { regex?: string; glob?: string }
import type { Paginated } from '~/types/responses'
export type EngineType = 'httpx' | 'selenium'
@ -29,7 +28,7 @@ export interface RequestConfig {
headers?: StringMap
params?: StringMap
data?: StringMap | string | null
json?: object | Array<unknown> | string | number | boolean | null
json_data?: object | Array<unknown> | string | number | boolean | null
timeout?: number
}
@ -79,31 +78,37 @@ export interface ParseConfig {
[field: string]: ExtractionRule | Container | undefined
}
export interface TaskDefinitionDocument {
name: string
match: Array<TaskMatchPattern>
export interface TaskDefinitionConfig {
parse: ParseConfig
priority?: number
engine?: EngineConfig
request?: RequestConfig
response?: ResponseConfig
}
// --- Summaries and Error Types ---
export interface TaskDefinitionDocument {
name: string
match_url: string[]
priority?: number
enabled?: boolean
definition: TaskDefinitionConfig
}
export type TaskDefinitionSummary = {
id: string,
name: string,
priority: number,
updated_at: number,
id: number
name: string
priority: number
match_url: ReadonlyArray<string>
enabled: boolean
created_at: string
updated_at: string
}
export type TaskDefinitionDetailed = TaskDefinitionSummary & {
definition: TaskDefinitionDocument
definition: TaskDefinitionConfig
}
export type TaskDefinitionList = Paginated<TaskDefinitionSummary>
export type TaskDefinitionErrorResponse = {
error: string,
}

174
ui/tests/composables/useTaskDefinitions.test.ts
View file

@ -1,27 +1,37 @@
import * as utils from '~/utils/index'
import { describe, it, expect, beforeEach, vi } from 'vitest'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { useTaskDefinitions } from '~/composables/useTaskDefinitions'
import type { TaskDefinitionSummary } from '~/types/task_definitions'
import type { TaskDefinitionDetailed, TaskDefinitionSummary } from '~/types/task_definitions'
vi.mock('~/composables/useNotification', () => {
const success = vi.fn()
const error = vi.fn()
return {
useNotification: () => ({ success, error }),
default: () => ({ success, error })
default: () => ({ success, error }),
}
})
// Sample data
const summary: TaskDefinitionSummary = {
id: 'abc',
id: 1,
name: 'Test',
priority: 1,
updated_at: 123456,
}
const listPayload = (items: TaskDefinitionSummary[]) => ({
items,
pagination: {
page: 1,
per_page: 50,
total: items.length,
total_pages: 1,
has_next: false,
has_prev: false,
},
})
// Helper to create a mock Response object
function createMockResponse({ ok, status, jsonData }: { ok: boolean, status: number, jsonData: any }) {
return {
ok,
@ -43,32 +53,31 @@ function createMockResponse({ ok, status, jsonData }: { ok: boolean, status: num
}
describe('useTaskDefinitions', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('sorts definitions by priority then name', async () => {
const items = [
{ id: '1', name: 'B', priority: 2, updated_at: 1 },
{ id: '2', name: 'A', priority: 2, updated_at: 2 },
{ id: '3', name: 'C', priority: 1, updated_at: 3 },
{ id: 1, name: 'B', priority: 2, updated_at: 1 },
{ id: 2, name: 'A', priority: 2, updated_at: 2 },
{ id: 3, name: 'C', priority: 1, updated_at: 3 },
]
vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({
ok: true,
status: 200,
jsonData: items,
jsonData: listPayload(items),
}))
const defs = useTaskDefinitions()
await defs.loadDefinitions()
expect(defs.definitions.value.map(d => d.id)).toEqual(['3', '2', '1'])
expect(defs.definitions.value.map(d => d.id)).toEqual([3, 2, 1])
})
it('handles empty payload', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({
ok: true,
status: 200,
jsonData: [],
jsonData: listPayload([]),
}))
const defs = useTaskDefinitions()
await defs.loadDefinitions()
@ -76,115 +85,66 @@ describe('useTaskDefinitions', () => {
expect(defs.lastError.value).toBeNull()
})
it('handles malformed payload', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce({
ok: true,
json: async () => ({ jsonData: [{}] })
})
const defs = useTaskDefinitions()
await expect(defs.loadDefinitions()).resolves.toBeUndefined()
})
it('handles malformed payload (throws when throwInstead is true)', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce({
ok: true,
json: async () => ({ jsonData: [{}] })
})
it('throws loadDefinitions error when throwInstead is true', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({
ok: false,
status: 500,
jsonData: { error: 'Server error' },
}))
const defs = useTaskDefinitions()
defs.throwInstead.value = true
await expect(defs.loadDefinitions()).rejects.toThrow()
await expect(defs.loadDefinitions()).rejects.toThrow('Server error')
})
it('handles duplicate IDs (no deduplication, both present)', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce({
ok: true,
json: async () => [
{ id: 'dup', name: 'A', priority: 1 },
{ id: 'dup', name: 'B', priority: 2 },
]
})
const defs = useTaskDefinitions()
await defs.loadDefinitions()
expect(defs.definitions.value.length).toBe(2)
expect(defs.definitions.value[0].name).toBe('A')
expect(defs.definitions.value[1].name).toBe('B')
})
it('handles error on getDefinition', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce({
it('returns null on getDefinition error', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({
ok: false,
status: 404,
statusText: 'Not Found',
json: async () => ({})
})
jsonData: { error: 'Not Found' },
}))
const defs = useTaskDefinitions()
await expect(defs.getDefinition('notfound')).rejects.toThrow('Request failed with status 404')
defs.throwInstead.value = false // Reset from previous test
const result = await defs.getDefinition(123)
expect(result).toBeNull()
expect(defs.lastError.value).toBe('Not Found')
})
it('handles malformed getDefinition response', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce({
ok: true,
json: async () => ({})
})
const defs = useTaskDefinitions()
await expect(defs.getDefinition('bad')).rejects.toThrow('Task definition response is missing definition payload.')
})
it('handles error on createDefinition', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce({
ok: false,
status: 400,
statusText: 'Bad Request',
json: async () => ({})
})
const defs = useTaskDefinitions()
await expect(defs.createDefinition({ id: 'fail', name: 'Fail', priority: 1 })).rejects.toThrow('Request failed with status 400')
})
it('handles error on updateDefinition', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce({
ok: false,
status: 400,
statusText: 'Bad Request',
json: async () => ({})
})
const defs = useTaskDefinitions()
await expect(defs.updateDefinition({ id: 'fail', name: 'Fail', priority: 1 })).rejects.toThrow('Request failed with status 400')
})
it('handles error on deleteDefinition', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce({
ok: false,
status: 400,
statusText: 'Bad Request',
json: async () => ({})
})
const defs = useTaskDefinitions()
await expect(defs.deleteDefinition('fail')).rejects.toThrow('Request failed with status 400')
})
it('loads definitions successfully (duplicate test)', async () => {
it('calls success notification on createDefinition', async () => {
const payload: TaskDefinitionDetailed = {
id: 2,
name: 'New',
priority: 0,
updated_at: 999,
definition: { name: 'New', match: ['https://example.com'], parse: { link: { type: 'css', expression: 'a' } } },
}
vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({
ok: true,
status: 200,
jsonData: [summary],
jsonData: payload,
}))
const defs = useTaskDefinitions()
await defs.createDefinition(payload.definition)
const notify = (await import('~/composables/useNotification')).useNotification()
expect(notify.success).toHaveBeenCalledWith('Task definition created.')
expect(defs.definitions.value.some(item => item.id === payload.id)).toBe(true)
})
it('removes definition on deleteDefinition', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({
ok: true,
status: 200,
jsonData: listPayload([summary]),
}))
const defs = useTaskDefinitions()
await defs.loadDefinitions()
expect(defs.definitions.value).toEqual([summary])
expect(defs.lastError.value).toBeNull()
})
it('calls success notification on createDefinition', async () => {
vi.spyOn(utils, 'request').mockResolvedValueOnce({
vi.spyOn(utils, 'request').mockResolvedValueOnce(createMockResponse({
ok: true,
json: async () => ({})
})
const defs = useTaskDefinitions()
await defs.createDefinition({ id: 'new', name: 'New', priority: 1 })
// Access the spy directly from the mock
const notify = (await import('~/composables/useNotification')).useNotification()
expect(notify.success).toHaveBeenCalledWith('Task definition created.')
status: 200,
jsonData: { status: 'deleted' },
}))
const result = await defs.deleteDefinition(summary.id)
expect(result).toBe(true)
expect(defs.definitions.value).toEqual([])
})
});
})